{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import log_loss\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>40.7145</td>\n",
       "      <td>-73.9425</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>40.7947</td>\n",
       "      <td>-73.9667</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>40.7388</td>\n",
       "      <td>-74.0018</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>40.7539</td>\n",
       "      <td>-73.9677</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>40.8241</td>\n",
       "      <td>-73.9493</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 225 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  latitude  longitude  price  price_bathrooms  \\\n",
       "0        1.5         3   40.7145   -73.9425   3000           1200.0   \n",
       "1        1.0         2   40.7947   -73.9667   5465           2732.5   \n",
       "2        1.0         1   40.7388   -74.0018   2850           1425.0   \n",
       "3        1.0         1   40.7539   -73.9677   3275           1637.5   \n",
       "4        1.0         4   40.8241   -73.9493   3350           1675.0   \n",
       "\n",
       "   price_bedrooms  room_diff  room_num  Year       ...        walk  walls  \\\n",
       "0      750.000000       -1.5       4.5  2016       ...           0      0   \n",
       "1     1821.666667       -1.0       3.0  2016       ...           0      0   \n",
       "2     1425.000000        0.0       2.0  2016       ...           0      0   \n",
       "3     1637.500000        0.0       2.0  2016       ...           0      0   \n",
       "4      670.000000       -3.0       5.0  2016       ...           0      0   \n",
       "\n",
       "   war  washer  water  wheelchair  wifi  windows  work  interest_level  \n",
       "0    0       0      0           0     0        0     0               1  \n",
       "1    0       0      0           0     0        0     0               2  \n",
       "2    0       0      0           0     0        0     0               0  \n",
       "3    0       0      0           0     0        0     0               2  \n",
       "4    1       0      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 225 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('RentListingInquries_FE_train.csv')\n",
    "test = pd.read_csv('RentListingInquries_FE_test.csv')\n",
    "predict_id = test['listing_id'].values\n",
    "test = test.drop('listing_id', axis=1)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 49352 entries, 0 to 49351\n",
      "Columns: 225 entries, bathrooms to interest_level\n",
      "dtypes: float64(7), int64(218)\n",
      "memory usage: 84.7 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>49352.00000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>4.935200e+04</td>\n",
       "      <td>4.935200e+04</td>\n",
       "      <td>4.935200e+04</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.0</td>\n",
       "      <td>...</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "      <td>49352.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.21218</td>\n",
       "      <td>1.541640</td>\n",
       "      <td>40.741545</td>\n",
       "      <td>-73.955716</td>\n",
       "      <td>3.830174e+03</td>\n",
       "      <td>1.697863e+03</td>\n",
       "      <td>1.657567e+03</td>\n",
       "      <td>-0.329460</td>\n",
       "      <td>2.753820</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003080</td>\n",
       "      <td>0.000385</td>\n",
       "      <td>0.186477</td>\n",
       "      <td>0.009361</td>\n",
       "      <td>0.000446</td>\n",
       "      <td>0.028165</td>\n",
       "      <td>0.002026</td>\n",
       "      <td>0.001013</td>\n",
       "      <td>0.000952</td>\n",
       "      <td>1.616895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.50142</td>\n",
       "      <td>1.115018</td>\n",
       "      <td>0.638535</td>\n",
       "      <td>1.177912</td>\n",
       "      <td>2.206687e+04</td>\n",
       "      <td>1.100477e+04</td>\n",
       "      <td>7.817996e+03</td>\n",
       "      <td>0.947732</td>\n",
       "      <td>1.446091</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.055412</td>\n",
       "      <td>0.019618</td>\n",
       "      <td>0.389495</td>\n",
       "      <td>0.101625</td>\n",
       "      <td>0.021109</td>\n",
       "      <td>0.165446</td>\n",
       "      <td>0.044969</td>\n",
       "      <td>0.031814</td>\n",
       "      <td>0.030846</td>\n",
       "      <td>0.626035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-118.271000</td>\n",
       "      <td>4.300000e+01</td>\n",
       "      <td>2.150000e+01</td>\n",
       "      <td>4.300000e+01</td>\n",
       "      <td>-5.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>40.728300</td>\n",
       "      <td>-73.991700</td>\n",
       "      <td>2.500000e+03</td>\n",
       "      <td>1.225000e+03</td>\n",
       "      <td>1.066667e+03</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>40.751800</td>\n",
       "      <td>-73.977900</td>\n",
       "      <td>3.150000e+03</td>\n",
       "      <td>1.500000e+03</td>\n",
       "      <td>1.383417e+03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>40.774300</td>\n",
       "      <td>-73.954800</td>\n",
       "      <td>4.100000e+03</td>\n",
       "      <td>1.850000e+03</td>\n",
       "      <td>1.962500e+03</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>10.00000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>44.883500</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.490000e+06</td>\n",
       "      <td>2.245000e+06</td>\n",
       "      <td>1.496667e+06</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>13.500000</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 225 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         bathrooms      bedrooms      latitude     longitude         price  \\\n",
       "count  49352.00000  49352.000000  49352.000000  49352.000000  4.935200e+04   \n",
       "mean       1.21218      1.541640     40.741545    -73.955716  3.830174e+03   \n",
       "std        0.50142      1.115018      0.638535      1.177912  2.206687e+04   \n",
       "min        0.00000      0.000000      0.000000   -118.271000  4.300000e+01   \n",
       "25%        1.00000      1.000000     40.728300    -73.991700  2.500000e+03   \n",
       "50%        1.00000      1.000000     40.751800    -73.977900  3.150000e+03   \n",
       "75%        1.00000      2.000000     40.774300    -73.954800  4.100000e+03   \n",
       "max       10.00000      8.000000     44.883500      0.000000  4.490000e+06   \n",
       "\n",
       "       price_bathrooms  price_bedrooms     room_diff      room_num     Year  \\\n",
       "count     4.935200e+04    4.935200e+04  49352.000000  49352.000000  49352.0   \n",
       "mean      1.697863e+03    1.657567e+03     -0.329460      2.753820   2016.0   \n",
       "std       1.100477e+04    7.817996e+03      0.947732      1.446091      0.0   \n",
       "min       2.150000e+01    4.300000e+01     -5.000000      0.000000   2016.0   \n",
       "25%       1.225000e+03    1.066667e+03     -1.000000      2.000000   2016.0   \n",
       "50%       1.500000e+03    1.383417e+03      0.000000      2.000000   2016.0   \n",
       "75%       1.850000e+03    1.962500e+03      0.000000      4.000000   2016.0   \n",
       "max       2.245000e+06    1.496667e+06      8.000000     13.500000   2016.0   \n",
       "\n",
       "            ...                walk         walls           war        washer  \\\n",
       "count       ...        49352.000000  49352.000000  49352.000000  49352.000000   \n",
       "mean        ...            0.003080      0.000385      0.186477      0.009361   \n",
       "std         ...            0.055412      0.019618      0.389495      0.101625   \n",
       "min         ...            0.000000      0.000000      0.000000      0.000000   \n",
       "25%         ...            0.000000      0.000000      0.000000      0.000000   \n",
       "50%         ...            0.000000      0.000000      0.000000      0.000000   \n",
       "75%         ...            0.000000      0.000000      0.000000      0.000000   \n",
       "max         ...            1.000000      1.000000      1.000000      2.000000   \n",
       "\n",
       "              water    wheelchair          wifi       windows          work  \\\n",
       "count  49352.000000  49352.000000  49352.000000  49352.000000  49352.000000   \n",
       "mean       0.000446      0.028165      0.002026      0.001013      0.000952   \n",
       "std        0.021109      0.165446      0.044969      0.031814      0.030846   \n",
       "min        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "25%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "50%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "75%        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "max        1.000000      1.000000      1.000000      1.000000      1.000000   \n",
       "\n",
       "       interest_level  \n",
       "count    49352.000000  \n",
       "mean         1.616895  \n",
       "std          0.626035  \n",
       "min          0.000000  \n",
       "25%          1.000000  \n",
       "50%          2.000000  \n",
       "75%          2.000000  \n",
       "max          2.000000  \n",
       "\n",
       "[8 rows x 225 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = train['interest_level']\n",
    "train = train.drop(['interest_level'], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.preprocessing import StandardScaler\n",
    "# ss_X = StandardScaler()\n",
    "# X_train = ss_X.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Logistic Regression without Standardization\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is: [ 0.68688766  0.67828085  0.67589458  0.67203656  0.68522762]\n",
      "cv logloss is: 0.679665451271\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "print('logloss of each fold is:', -loss)\n",
    "print('cv logloss is:', -loss.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Logistic Regression with Normalization\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "penaltys = ['l1', 'l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty=penaltys, C=Cs)\n",
    "\n",
    "lr_penalty = LogisticRegression()\n",
    "grid = GridSearchCV(lr_penalty, tuned_parameters, cv=5, scoring='neg_log_loss')\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([   5.41882091,    2.10660062,    8.06479168,    2.60537887,\n",
       "          17.43303895,    2.3050508 ,   18.79704456,    2.17802167,\n",
       "         202.05776668,    2.25535836,  202.49067483,    2.16437426,\n",
       "         239.14329519,    2.50130601]),\n",
       " 'mean_score_time': array([ 0.00855818,  0.01216059,  0.0084609 ,  0.01193399,  0.00816326,\n",
       "         0.01146746,  0.00790429,  0.01236591,  0.00839009,  0.01126399,\n",
       "         0.00801821,  0.01161203,  0.00799842,  0.01161661]),\n",
       " 'mean_test_score': array([-0.71291071, -0.68616647, -0.68691917, -0.67780828, -0.67524252,\n",
       "        -0.67758521, -0.67344222, -0.67966511, -0.67723268, -0.6786923 ,\n",
       "        -0.68037772, -0.67986452, -0.68272451, -0.67695346]),\n",
       " 'mean_train_score': array([-0.71226786, -0.68476671, -0.68593865, -0.67591827, -0.67231091,\n",
       "        -0.67550867, -0.66609598, -0.67780669, -0.66395779, -0.67663659,\n",
       "        -0.66368431, -0.67821154, -0.66356872, -0.67466441]),\n",
       " 'param_C': masked_array(data = [0.001 0.001 0.01 0.01 0.1 0.1 1 1 10 10 100 100 1000 1000],\n",
       "              mask = [False False False False False False False False False False False False\n",
       "  False False],\n",
       "        fill_value = ?),\n",
       " 'param_penalty': masked_array(data = ['l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2' 'l1' 'l2'],\n",
       "              mask = [False False False False False False False False False False False False\n",
       "  False False],\n",
       "        fill_value = ?),\n",
       " 'params': [{'C': 0.001, 'penalty': 'l1'},\n",
       "  {'C': 0.001, 'penalty': 'l2'},\n",
       "  {'C': 0.01, 'penalty': 'l1'},\n",
       "  {'C': 0.01, 'penalty': 'l2'},\n",
       "  {'C': 0.1, 'penalty': 'l1'},\n",
       "  {'C': 0.1, 'penalty': 'l2'},\n",
       "  {'C': 1, 'penalty': 'l1'},\n",
       "  {'C': 1, 'penalty': 'l2'},\n",
       "  {'C': 10, 'penalty': 'l1'},\n",
       "  {'C': 10, 'penalty': 'l2'},\n",
       "  {'C': 100, 'penalty': 'l1'},\n",
       "  {'C': 100, 'penalty': 'l2'},\n",
       "  {'C': 1000, 'penalty': 'l1'},\n",
       "  {'C': 1000, 'penalty': 'l2'}],\n",
       " 'rank_test_score': array([14, 12, 13,  6,  2,  5,  1,  8,  4,  7, 10,  9, 11,  3], dtype=int32),\n",
       " 'split0_test_score': array([-0.71800453, -0.69354586, -0.69449347, -0.68563673, -0.68343884,\n",
       "        -0.68890832, -0.68116069, -0.68688766, -0.68441689, -0.68769588,\n",
       "        -0.68741269, -0.68933513, -0.69068783, -0.68911852]),\n",
       " 'split0_train_score': array([-0.71198839, -0.68145975, -0.684504  , -0.67135344, -0.67122995,\n",
       "        -0.67532964, -0.66513044, -0.6722876 , -0.66323127, -0.6738555 ,\n",
       "        -0.66292354, -0.67542509, -0.66262012, -0.67547661]),\n",
       " 'split1_test_score': array([-0.70829856, -0.6854324 , -0.68271197, -0.68109986, -0.67240974,\n",
       "        -0.6744358 , -0.67160441, -0.67828085, -0.67687132, -0.67483725,\n",
       "        -0.68140132, -0.67684661, -0.68445196, -0.67636672]),\n",
       " 'split1_train_score': array([-0.7088629 , -0.68591189, -0.68351188, -0.68178503, -0.67035813,\n",
       "        -0.67454881, -0.66389694, -0.67874622, -0.66153937, -0.67426306,\n",
       "        -0.66126747, -0.6773138 , -0.66126519, -0.67762761]),\n",
       " 'split2_test_score': array([-0.71354666, -0.68515054, -0.68615807, -0.67638341, -0.6757139 ,\n",
       "        -0.67716553, -0.67499011, -0.67589458, -0.67839461, -0.67909524,\n",
       "        -0.68018834, -0.68110827, -0.68060565, -0.67409094]),\n",
       " 'split2_train_score': array([-0.71360426, -0.68580697, -0.68607475, -0.67549722, -0.67196131,\n",
       "        -0.67659559, -0.66558016, -0.67536994, -0.66345381, -0.67838877,\n",
       "        -0.66317409, -0.68069384, -0.66327133, -0.6716984 ]),\n",
       " 'split3_test_score': array([-0.70848555, -0.67825592, -0.68089542, -0.66903563, -0.66757786,\n",
       "        -0.66335857, -0.66445529, -0.67203656, -0.66850245, -0.66494482,\n",
       "        -0.67194129, -0.67153712, -0.67482456, -0.66439105]),\n",
       " 'split3_train_score': array([-0.71423366, -0.68803588, -0.68922447, -0.67960593, -0.67494353,\n",
       "        -0.67294284, -0.6694541 , -0.68387866, -0.66753929, -0.6756445 ,\n",
       "        -0.66698257, -0.68285375, -0.6667244 , -0.67392896]),\n",
       " 'split4_test_score': array([-0.71621925, -0.68844835, -0.69033793, -0.67688549, -0.67707282,\n",
       "        -0.68405981, -0.67500109, -0.68522762, -0.67797834, -0.68689083,\n",
       "        -0.68094513, -0.68049565, -0.68305263, -0.68080123]),\n",
       " 'split4_train_score': array([-0.71265009, -0.68261908, -0.68637813, -0.67134973, -0.67306163,\n",
       "        -0.67812647, -0.66641826, -0.67875101, -0.66402521, -0.68103114,\n",
       "        -0.66407387, -0.67477123, -0.66396257, -0.67459047]),\n",
       " 'std_fit_time': array([  1.86469327,   0.15408801,   1.25334034,   0.22197171,\n",
       "          1.46477686,   0.24588378,   1.87718542,   0.11725024,\n",
       "          3.22320504,   0.28354319,   4.45957339,   0.21163595,\n",
       "         67.24430664,   0.09093215]),\n",
       " 'std_score_time': array([ 0.00091433,  0.00118316,  0.00071059,  0.00028116,  0.00062672,\n",
       "         0.0001733 ,  0.00037245,  0.0020333 ,  0.00075087,  0.0002475 ,\n",
       "         0.00057139,  0.0003469 ,  0.00046655,  0.0001839 ]),\n",
       " 'std_test_score': array([ 0.00395356,  0.00497626,  0.00497247,  0.00551306,  0.00524463,\n",
       "         0.00874798,  0.00545072,  0.00561133,  0.00509655,  0.00839403,\n",
       "         0.0049412 ,  0.00583269,  0.00516552,  0.00811372]),\n",
       " 'std_train_score': array([ 0.00186913,  0.00239288,  0.00194708,  0.00424038,  0.00158734,\n",
       "         0.00176472,  0.00186628,  0.00387405,  0.00197293,  0.00271052,\n",
       "         0.00188205,  0.00310219,  0.00181152,  0.00193824])}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.673442223536\n",
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEKCAYAAAA1qaOTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3Xd4VFX6wPHvmZnMpPcKIYQk9CqEosCKFRYLRcFeUJd117K6yqpbRHRV+Km77q4dXRfUFextraAioIJ0AgKht/RCepmZ8/vjThokMMBMZgLv53ny3Dv3njvzJoT75txz7nuV1hohhBDiZJl8HYAQQohTgyQUIYQQHiEJRQghhEdIQhFCCOERklCEEEJ4hCQUIYQQHiEJRQghhEdIQhFCCOERklCEEEJ4hMUXH6qUigYWAqnAbmCq1rqklXYpwMtAF0AD47XWu5VSCvgrMAVwAM9rrf95rM+NjY3VqampHvouhBDi9LB69epCrXXcsdr5JKEA9wOLtdazlVL3u17f10q7+cCjWuuvlFKhgNO1/UaMJNNLa+1USsW786GpqamsWrXq5KMXQojTiFJqjzvtfHXJawIwz7U+D5h4eAOlVB/AorX+CkBrXaG1rnLt/g3wsNba6dqX7/2QhRBCHI2vEkqC1jrHtZ4LJLTSpgdQqpR6Tym1Vin1hFLK7NqXDlyhlFqllPpMKdW9PYIWQgjRNq9d8lJKLQISW9n1p+YvtNZaKdVayWMLMBo4A9iLMeZyI/AKYANqtNaZSqnJwL9dbVuLYzowHSAlJeWEvhchhBDH5rWEorU+v619Sqk8pVSS1jpHKZUEtHbJaj+wTmu903XMB8AIjISyH3jP1e594NWjxPES8BJAZmam1OoXQggv8dUlr4+AG1zrNwAfttLmJyBSKdUws+BcYLNr/QPgHNf62cA2L8UphBDCTb5KKLOBC5RS2cD5rtcopTKVUi8DaK0dwL3AYqXURkABc5sdf5lr++PALe0cvxBCiMOo0+mJjZmZmVqmDQshxPFRSq3WWmceq53cKS+EEMIjJKEIIURrXr3I+Oro2vH7kIQihPCcU+UkLE6IJBQhhBAeIQlFCCGER0hCEUII4RGSUIQQ4lSmddOXl/mqfL0QQhyb1uB0gLMeHPXgtDctnfXgsDfbd/hre7Pth78+rF1r+4q2Axo+vN11QnaAdrbxpZvWna21061scxxjf8P7tfWZhx/bRnwNCrMhrodX/7kkoQghTpzWUJ5jnHwLs6F4p3Fyfufmlifn5ongmCf/w5JEe1EmMAWAOQBMFqirBKVg+2JjnzKBydS0fsSXarZuPnK/yQwqoJVtrR1/Il+qlfc0wbo3je8vONrrP0JJKEKIY6sth6IdTYmjaDsUZRvb6iqa2jWclA+sdp2YA8BsaXmiDghq+bqtdg3bTZa29x31PY7ynuYA48TbvK3psBGAhunP0/7Xfj9nb9i93FiGxHr9oyShCOFr/nLictjh0F4obEgWzZJHeU6zhgoiu0BMd+gyAmK7Q0yGsXx3uvGXsq+/F+ETklCEON1UFhkJo7Gn4UocJbvAUdfULjDSSBJpY5oSRkx3iO5m9DJao1R7fAfCT0lCEeJUVF9jjGc09jSa9TqqS5ramQIgOs1IGD3HGQmjoccRHCMJQhwXSShCdFROJ5QfPLKnUZQNpfuAZtNEw5KMJNFnYlNPIyYdIrsa4wxCeID8Jgnh72rKmgbAGxJG4XYo3gH1VU3tAkKMJJE8FAZe7Uoc6UYisYX5Ln7hW+04niUJRQh/oJ2uy1Lbm41v7DDWK/Ka2imT0auIyYBuo41lw/hGWJJcovIkmVhw3CShCOErWsOu7yB/szGu8cyQpn3BMUaiyLgAYl1Jo2FA3GLzXczHIifh05okFCHaW30NZL0DPz4PeVnG/RFhSXDun5sNiHv/JjQhPE0SihDtpSIffnoFVr0ClQUQ3wcu/ZdxJ7MywRnX+DpCIU6KJBQhvC13o9Eb2fi2cZ9H97Ew4jfG/R1KwfqFvo5QCI+QhCKENzidsO1z+PE52L0UAoJh8PUw/FbjspYQpyBJKEJ4Um0FrHsDVrxg3FgY3hnOn2UkExkXEac4SShCeELpXlj5EqyeD7WHoHMmXP5n6H2pUXxQiNOAJBQhTpTWsG8l/Pgs/PwxoKDPBBjxW+gy1NfRCdHuJKEIcbwc9bD5Q2N85MBqCIyAs+6Aob8yqvAepyvq/gyADM2Ljk4SihDuqiqG1f+BlXONGlrR6TD+SRh4FdhCfR2dED4nCUWIYynYBiueN+4XsVdDt7PhkqeNu9gPfyiTEKcxnyQUpVQ0Rg8/FdgNTNVal7TSLgV4GeiCUTp1vNZ6t1JqKdBQ7S4eWKm1ntgOoQs/Me3zaQC8Ou5V73yA1rDzG/jhOdj+FZhtMGCKMT6S0Nc7nymEF3j9/0ozvuqh3A8s1lrPVkrd73p9Xyvt5gOPaq2/UkqFAk4ArfXohgZKqXeBD9shZnE6qK+GDW8ZNyIW/Awh8TDmj5B5E4TG+To6IfyarxLKBGCMa30e8C2HJRSlVB/AorX+CkBrXcFhlFLhwLnANC/GKk4H5bnw08uw6t9QVQQJ/WHi89DvMv8uxiiEH/FVQknQWjc8pDoXSGilTQ+gVCn1HtANWATcr7V2NGszEaOnU+bVaMWp6+A6ozeS9S447dDzl8ZlrdRRUgpeiOPktYSilFoEJLay60/NX2ittVJKt9LOAowGzgD2Yoy53Ai80qzNVRhjLEeLYzowHSAlJcXN6IW/25xzEn9DOB2w9VMjkexZDtZQGHozDJtuPJBKCHFCvJZQtNbnt7VPKZWnlErSWucopZKA/Faa7QfWaa13uo75ABiBK6EopWKBYcCkY8TxEvASQGZmZmuJS5wuaspg7etGWZTSPRCRAhf+Fc64DoIifR3dKaE9B4CF//HVJa+PgBuA2a5la4PqPwGRSqk4rXUBxljJqmb7Lwc+0VrXeDtY4X+61u9wv3HJbljxIqx5DerKocsIuOBh6HWxPE9dCA/y1f+m2cBbSqmbgT3AVAClVCZwq9b6Fq21Qyl1L7BYKaWA1cDcZu9xpet9hDiS1rD3B/jhWePyljJB30lG2fjOQ459vBDiuPkkoWiti4DzWtm+Cril2euvgAFtvMcYb8UnOjB7HWx636ivlbMegqJg5F0w7FcQ3snX0QlxSpP+vjg1VBbB6n/DypehIhdie8DFf4cBV4I12NfRCXFakIRymtlz3fUAdH1tvo8j8ZD8n43ZWhsWgr0G0s+FCc8aSymLIkS7koQiOqQYh51flxbCcyPAEggDrzSehhjf29ehndZOajq36PAkoZxmthRvAaCrj+M4KcW7mFWYQ6jTAef+GYbcBCExvo5KiNOeXBMQHUthNrw6niCnk1kxSfCLGZJMhPAT0kMRHUfeJpg/AbRmVmwSewOsvo5ICL935b82GSvjvP9Z0kMRHcPBdfCfi8BkgWmfdehkYnc42Vdcxfc7Cnnrp33sL6lib3EVS7YVUF3nOPYbCOGnpIci/N++n+D1yyAwHG74CKLTfB3RUTmdmrzyGvaXVLOvuIp9xdXsL6liX0kV+0uqyTlUg8PZsgqQAm7490qsZhNDukYxqnssozJi6dc5ArNJilSKjkESivBvu5fBf6+AkDi44ePGZ7bf+FadsX96+4ektaawoq4xQewrrmJ/s/WDpTXUOZwtjokPs9ElOpghXaPoEhVMl+ggkqOC6RIVzD1vr0NruOO87izLLmDZ9iKe+GIrT3yxlYigAM5Kj2FU91hGZ8SREiP31Aj/JQlF+K/ti2HBNUYSuf4jCE9q3FWrgrz2sVprDlXXs6+42pU0mvcyjGVNfcuEERNiJTkqiL6dIxjbL9GVNIJJjgqic2QQgQHmNj/v6nfvAODs36zm7B7GQ7wKymv5fkchy7ILWba9kM+ycgHoEh3EqIw4RneP5az0GCKD/evS3/3/3W2snAJPKJJCl8dPEorwT1s/g7euN+54v+4Djz8tsbymvumSlCtJNCSN/SXVVNTaW7QPD7TQJTqY9LgQxvSIIzkqyJUwjKQRYvPsf6W4MBsTBnVmwqDOaK3ZWVjJsuxClmYX8vH6g7y5ci9KQf/OEYzMiGV0RixDUqOwWdpOXOL0pJwaa60DrTXKy8/4kYQi/M+mD+DdmyGxP1z7HgRHH/dbVNc5mi5DNbsc1bBeWlXfon2w1UwXV3IYkRZDcpTrkpTr0lREUICnvrvjppQiPS6U9LhQbjgrFbvDyfr9pSzLLmLZ9gLmfreT57/dQWCAiaGp0YzuHsuojDh6JYZhkvGX046zpobqdeupWrmSqpUrSd5VgQLqtm/H1r27Vz9bEorwL+sXwge3QvJQuOZtCIxos2mxNYLvthUckTT2l1RRWFHXoq3VYjJ6FVHBDEyOpEt0cGMC6RIdTFRwgNf/evMUi9nEkK7RDOkaze/O705FrZ0VO4tYml3I8u2FPPbpFmALMSFWRmYYg/ujusfSKdJ7lwmF7zhra1skkOr169F1dWAyEdi3L+WRAdQEWuiZnOz1WCShCP+xeh58/Dvj8btXLQBbKADa4UDX1aHr6nDW1uKorWN59AhWxgwm76Vl1FqsWEyKzlFBJEcFcX7vhGaXpIwkEhtqO2X/Wg+1WTivdwLn9TaepJ17qIZl243ksmx7IR+tPwhAWlyIkVwyYhmRHkN4oO96XR1Be96/cTyctbVUr19P1cqfjASybl1TAunTh6hrryV42FCChwzBHBbGtouGAWAK8v4fFJJQRKu01lBfj9N1Ite1tY0ndF1X7zrBN99WZ2xvXK9tdmzdYe3rGhNE4/uWHESXHkSbUtDWMvQLFzQej91+RHw3AzfzNVopTJ06E5yRji3K9ZWRjjU9CXNoaPv/4PxAYkQglw9J5vIhyWit2ZpX3ji4//aq/cz/YQ9mk2JgcgSjuscxKiOWM1IiCTDLbWnNVfnJPUHOujpq1q+ncuVKqlYclkB69ybqmmuMBJKZiTkszKexSkI5jdTn5RGdX4PF7mT31de0PKHXNzvxu7Z5hMmEstlQVivKGoDJ6lpvvq2+CLN9Hyo5EdXtLFRgICabDRXQrJ3NSmk9vLEmh4NVTgYWradn+U4GXDOduh07qN2+g6rvv0fXN42NWBISsKWnY01Px5bekGjSsURFeeZ76wCUUvRKDKdXYji3jE6jzu5kzd6SxgTzzNfZ/HNxNiFWMyPSYowB/u6xZMSHdphLgKcaZ10dNRs2tEwgtbWglJFArr6a4OHDjB5IeLivw21BEsppQDsclPz3TQqefprgqnrqbSZUQACm0BCU1dp0km880QcYJ3SrtdlJvdm2hpO91YrJZj0sQbi2N2yzHOVXTGtYMge+/Rj6TobJL4G59csw3+8o5LdvrIGUrjx3zWDybh1LWYiFuNtua3o7u536/fup3bmT2u07qNuxndodOyl99110VVVjO3N0NLa0NKwZ6djSM7Clp2FNz8ASH3fKn0StFhMj0mIYkRbDvWN7cqiqnh92GsllWXYhi7fkA5AQbmtMLiPTY4kPD/Rx5KcuZ10dNRs3UrliBVUrf6J67drGBGLr3YuoK68kePhwgjP9L4EcThLKKa5m82ZyHpxJTVYWIaNGsW3vWhwBJgbN+49vA9MaFj0Ey5+GgVfDhGfA1PqU19d+3MOsjzaRGhvCKzdk0jUmhPdbaacsFqypqVhTUwk799ymj3I6sefmUuvqydTtNJZln36Gs6yp3LopNNTo0WSkY0tr6tEEdOqEOkWfrRIRHMC4fkmM62fc47OvuIrl2wtZur2Qb7bk896aAwD0TAhrvHt/eFo0wdYjTx26rg6zw6gA0B5TVDsqXVdH9caNVK1cSeXKlVSvXYeuqTESSK+GBOLqgUS0PSnFH0lCOUU5Kysp+NczFM+fjzk6mk5PPUn4+PH8fPFwX4dmJJPP74cVL0DmTTD+qVYfhlXvcPLwx5t57cc9nNsrnn9cOYiwExhIViYTAZ06EdCpE6GjRzcLQ+MoLDQSzY4dxqWzHTupWPIdh959r+n4wECsad1cvZl0rOlp2NIzsKZ0OXoPrAPqEh3MlcNSuHJYCvaaWrZs2cu6jbvYujWbvWu/48OqcpbWV5IWUE8XUy2x9ipslWU4SkpwlpXRMCl166AzsCQmEJCQSEBSIpbDl4mJmCMjT4uko+vqqM7KapyFVbVmbcsEcsVUgoe5EkhkpMc/f8EdfXE4HVwo96GIE1H+zTfkPvII9oM5RF5xBfG/v9t//tJxOuGTu2DNPBjxWxj7GLTyS15SWcdv31jDDzuL+PUv0vjDuF4er2mllMISF4clLo6QESNa7HOUlrounW2nbsdOanfsoGr1Kso+/ripUUAAttSuWNMzsKWlNfZorKmpmGw2j8bqCc6aGhzFxdiLS3CUFDetFxdjLy7C0bBeYiydFRWYgSGur8b3MZmoCAylyBLCQWsIlcHRhPToQVyXRKqWfUxYfSV9Jl+DPTeH+tw8Kn/6CXtePjhaDnIrm81IOolJBCQmYGlcJhKQ2HGTjpFANjUlkLVr0dXVANh69SJy6hRChg0zBtG9kEBqHbVsK95GVlEWWYXGV42jhpzKHDqFdvL45zUnCeUUUp+XR96jj1H+5ZfYumfQ+b9vEDx4sK/DauKww4e3wYYFMPoeOPcvrSaT7Lxybpm/ipzSGp6aMpDLhnh//vzhzJGRBA8efMTPz1FRSd2unU09mu07qPl5M+VffmkkSwCTCWuXLo2TARp6NLa0bphCQjwWo7O6GntR8RHJwVFSbGxvlhwcxcU4m40jtWCxYImKwhwTgyU6iqDkZMzR0ViiozBHx2COjsISHY05ythmCg9HmUwUV9axfHsh67cbd/AfKK2GkQOJqCvjrE49GDwiisFdI+nbKQKbCeyFRdjzcqnPyT1i2WbSCQwkICEBS1KSa9mUbAJcX6aICJ8mHV1f7+qB/OTqgaxpSiA9exJ5+eWNs7A8PSHE7rSzo3QHWYVZbCraRFZhFtml2didxszI6MBobGYbUYFRBJi8P01cEsopQDsclLy5gIK//x1ttxN3993ETLsRZXWvzpNTO3E4Hdi1HYfTgUM7sDvtOLTjqNvbanf40qEd2O21ONa+hiN3A47Bk7DHd8axce4R7XYXlfPN1lzMwZoLBkaztno5q5Ye+f7LrjF+dTf+8DBdw7s2fiWHJhPQxsC+J5hDQwjq35+g/v1b/gxra6nbvbtFj6Z2x3Yqli6F5jPPOiW5kkvDpIB0TE6NUxmXKe0lJTiKirAXFxs9hhJXoigqwl5S3KIX0XDSOpwKCMAcHe1KCtFYu3Y1kkNUtJEcYmIak4M5OhpTWNgJnZCjQ6xcMrATlwzshNaaPUVVPHPHPewMTSHrYEJj/TGr2USfTuEMTolicNdYBg/v3upNltrhMJKOq2fTsKzPzcGem0flypXY81tJOkFBRrJp7NkYvR5LYgIBrkTkyaSj6+up2bSJyuYJxJWsbT16EHnZZUYCGTrUownEqZ3sLtvNpsJNjclja/FWahw1AIQFhNEntg839LmBfrH96BvTl8SQRG764iYA4oI9W76oNUprfexWp4jMzEy9atUqX4fhUTU//2wMum/cSMhZZ5H40EysKSlHtNNa8+2+b3n0g99RGqxRgYGNJ2iNb38HTMqEWZnR2kS9HczKQniQDavJgtlkxqzMWEwWLCYLZmXGbDKzpXAnoAkPDKK0trTxvczKTKfQTnQN70pqeCop4SmN64khiZhU+w6u6/p66vbta9Gjqd25g7qdu4zr6C5O2n44kbLZjOQQZSQAS0y0Kzk0JQWLK4GYo6MxhYT47C/29y8wLo5N+mo1+eU1rN1bypq9JazdU8r6/aXU2o1eXGJ4IIO7RjI4JYozUoxezNEKaDYwkk4h9tyWPZz6vFzsObnU5+UdPekkJRKQkNi4bBjPCUhMNHpezX5uDd/LxM9WuBKIMY23RQLp3t0Y/xg+zKMJRGvNwcqDjT2PTYWb2Fy0mYr6CgCCLEH0ju5Nn5g+9IvtR7/YfnQJ69Lq7/fwVy8DYMW0d084HqXUaq115rHaSQ+lg3JWVTUNukdG0unJJwm/aHyrJ5Ldh3Yz56c5LDuwjHgUw3eaSZ98VePJ2mwyY1EtT96tbW983cYxrR5nr8fyyV2Y9yzHfN5DmIfc2KKdWZmps2v++P5G3ltzgIv6J/HElAGtziJqruE/ydIr36W0ppQ95XvYW7aX3WW72VNmrK/OW021vemveKvJ2phgUsJTjIQTlkJqRCoxgTFeOQmrgABjfCUtDS64oHG7djqpP3iQ2u3bWXPfnZidmt6//l1TLyI62rgEFRWFCg7ucOMIAPFhgYztm8jYvomAMcni55wy1uwpYY0r0Xy6sbVejJFoWuvFKLOZgIQEAhISCBo4sNXP1XY79qIi7Dktezj1ubnYc3Op/PFHI+k4W1aMVsHBxnu7Jg/EldZhq3eybdjwxsuFtu4ZRE6caEzjHZqJJfr468y1pqCqoLHXkVWUxebCzZTUlgBgMVnoGdWTi9Iuom9MX/rG9iUtIg2Lyf9O3/4XkTim8m+/Jffhh41B9ylTiL/3nlYH3avqq3hxw4vM3zyfQHMgMzJnEHvfvzBrxdjHfu/9QGvL4b2rjGeaXPovGHzdEU3yy2v49WurWbu3lLvP78Gd52Uc98kzMjCSyMBIBsa1PMForSmoLmBP2Z4WX7sO7eK7/d9R72y6FBUSEGIkl/BUukZ0bbEebvX83H9lMmFNTsaanExxuHGJbtQtt3j8c/xJgNnEgORIBiRHcuNIY1t+eQ1r9pSydm8Ja/aW8MaKPfx7+S7g8F5MFP06h7tVTVlZLE1Jp4022m5v6um4vhp7ODk5VP7wA9HlduosioipE41eyNBMLDExJ/1zOFR7iE2Fm8gqympc5lcZ9/+YlIn0yHTGdBlD35i+9IvtR/eo7ljN/vWYgrZIQulA6vPyyXvsMcq/+AJrRjpd33id4CFDjminteazXZ/x1OqnyK/KZ0L6BO4achexQbF8oZ9pn2BrDsHrl8OB1XDZy9D/8iOaZB04xK/mr6K0qp7nrxnML/sntfJGJ04pRXxwPPHB8QxNHNpin8PpIKcy54hks7FwI1/s+QKnbvrrNcoW1WKcpuErJTyFIMupX3BRa021vZqyujIO1R5qXDZfb1huP68epWHl0geICowiOjCaKJtrGdi0DA0w7sSPDwtkXL9ExvUzejF1didbctvuxfTt7OrFuC6VnWjBS2WxNA7qt/UO758/GJRi0IN/OaHPAKisr+Tnop8bex+bijaxr3xf4/6u4V3JTMhsHPPoFd2L4ICO+xA1SSgdgHY4KFm4kIK//R1dV0fcXXcRc9O0VgfdtxZv5fGVj7M6bzV9YvrwtzF/O+Ivd6+rKobXJkHeJpjyH+hz6RFN/rchh3veXkd0sJV3fnMmfTu177Rms8lMclgyyWHJjOw8ssW+Okcd+8v3NyaZ3WW72Vu+lx8O/sCHOz5s0TYhOOGIsZqU8BSvTw44EXan/YgE0LisLTsyYdQZ2w/VHWqcNdQai8lCuDWcCFsEdgs4FazNX0txTXGLS47NBZgCWiSchvWGhNOlSxQDu0dzT2A6jvpgth60s26fkWBe/3EPryw7uV6MW46zp1zrqGVr8dYW4x47D+1sHKNMCkmiX2w/JnefTL/YfvSJ6eOV3q8v+SShKKWigYVAKrAbmKq1LmmlXQrwMtAF0MB4rfVupdR5wBMY45gVwI1a6+3tE337qtmyhZyZM6lZv4GQs84kceZMrF27HtHuUO0hnl33LAu3LiTcGs7MM2cyKWMS5jbuPveainyYPxGKtsOVb0CPsS12O52afyzO5h+LsxmcEsmL12USF+Zf92xYzVbSItNIizzy2fVV9VXsLXeN1Rza07j+5Z4vOVR7qLGdWZnpHNq5aazmKJMDgq3u/xtpramyVzWe6FtLDg3rDW0alpX1lUd979CAUCJsEYRbwwm3hZMRnEGELYIIawThtnAirBGN+5svgyxBjZcpmwblPweg2l5NSU0JJTUlFNcUU1JrrBfVFDVuL6kpYV/5Popriqmytz612WKyGImnUxSju0Vj0qHU1ARRUmFlVZGFL3cHoO2hWHQoPeOTyOzSiSFdYxjcNZKkCM/3Iuud9ewo3dHi0lV2STZ23TRdt19sP8amjqVvbF/6xPQhNijW43H4G1/1UO4HFmutZyul7ne9vq+VdvOBR7XWXymlQjEmwwA8D0zQWv+slPot8GfgxnaIu904q6ooePZZiv8zD3NEBJ2e+D/CL774iPEFp3byfvb7/GPNPzhUd4ipPaZy+xm3E2HzwY2MZQdh3qVQdgCueQvSxrTYXVVn55631vNZVi6XD0nm0Un9OtwTBoMDgukV3Yte0b2O2NcwOeDwy2jHmhxQ0c1OaI2ibsfHx7ysVFZb1njSao3FZGlx4k8ITqBHVI/GJNE8OTRPEmHWMI8M8s6+OhWASa7XQZYggkKD3L6hrtZR25h8imuKmxJRTQkltSUUVxdTXFtMSc0BSmpKqHBUQAQENft13wXsLDKxMD8E/UMIVhVGdGA0ncJiSYtOpGdcIvEhMU2X4WzRhNvC25wB2Np03S3FW6h11ALNpuv2vaFxxlVCcEKHnEhxsnyVUCYAY1zr84BvOSyhKKX6ABat9VcAWuuKZrs10NBXjAAOejHWdlexZAm5Dz9C/YEDRE65nPh77mn1jtoNBRt4bMVjbCraxOD4wfxx+B/pGd3TBxEDpXth3iVQWQTXvgtdz2qx+0BpNb+at4otuWX8aXxvbhnd7ZT7D3c8kwN2l+1m16FdLNm/BPtQV4JY9sfGY8ICwgi3hTf2AhKCE4zeQis9hObL5r2FjshmtpEYkkhiSKJb7escdU0Jp6aE4tpiiquLKawuZldxHvvKCiioKqaofgf5JetZX1ZjXBM5jFmZibBFtLjsVjTcTmGYZs6bZzX27hqm607tObVx0Lyt6bqnI18llAStdY5rPRdIaKVND6BUKfUe0A1YBNyvtXYAtwCfKqWqgTJgRCvHdzj1+fnkPfY45Z9/jjWuXjudAAAgAElEQVQ9na6vv0Zw5pFTvwurC3l69dN8uOND4oPimT16NuO7tT5luF0U7TB6JnXlcP0HkNwy5tV7ivn1a6uprXfyyo1DOadnvG/i9JFjTQ5YeOVwKm1w4bPvEmGNINQa6pdTQv2R1Wx1OwHll9Wwcnc+P+7Zx7oD+8kuzsVBBcpcQVhILdbwOrSphhJdRW5lHoVdnERWKi5Ou9jvp+v6C6/9ZJRSi4DW/pX/1PyF1lorpVq7s84CjAbOAPZijLncCLwC3I0xnrJCKTUD+BtGkmktjunAdICUVm748wfa6aR04ULyn/qbMej+uzuJufnmIwbd6531LNiygOfWPUeNo4ab+t3E9AHTCQlwv5zHgjv6AjD2GO3clr8F5k8AZz3c8AkkDWix+53V+/njextJigxkwfRMMuJ9+wAgf2M2mYmpNBFTacz4Ed4THx7IxQNSuHiAcR6oszvZ7LovZu2+UtbsKSG71Lg0aTWb6HRoNzHVuSQPv4YEFUakJRSz6liXaNub1xKK1vr8tvYppfKUUkla6xylVBKQ30qz/cA6rfVO1zEfACOUUh8BA7XWK1ztFgKfHyWOl4CXwLhT/sS+G++p2bqV3AdnUr1+PcFnjiBp5kysqalHtFuRs4LZK2ezvXQ7IzuP5P6h95MacWS7dpW70RiAN5nhxv9BfO/GXQ6nZvZnPzN36S5GZsTw7NWDiQz23Fz6Pkmn1uwY0f6sFhODukQyqEvT5eS8shrXPTGlfPW/XayN6s/3H29u3B8ZHED3+FC6J4TRPT6UHq5lXJitQ19q9BRf9d0+Am4AZruWH7bS5icgUikVp7UuAM4FVgElQIRSqofWehtwAfBz+4TtOc7qagqfe46iV/+DOSyMTv83h/BLLjnilzKnIocnVz3Jl3u+pHNoZ/55zj8Z02WM7395D6yG1yaDNQRu+Bhi0ht3ldXUc+eba/l2awHXn9mVv1zcRx4vKzqEhPDAxufD9P77tWhg1LvL2ZZXQXZ+OdvyKtieX87/NuRwqLrpxtiIoFYSTUIo8adZovFVQpkNvKWUuhnYA0wFUEplArdqrW/RWjuUUvcCi5XxL7IamKu1tiulfgW8q5RyYiSYm3zzbZyYiqVLyX1oFvUHDhBx+WXE33PPETWAah21zNs0j7kb5gJw26DbmNZvGjazH0yx3fsjvDEFgqLgho8gKrVx167CSm6Z9xN7iqp4dFI/rhkul3FEx6UwLpXFhwcyqnvTtF+tNQUVtWTnVZCdV862/Aq251XwWVYOb1Y1JZrwQAvdE8LokRBKRnxTskkIPzUTjU8Sita6CDivle2raDYW4prhNaCVdu9Dqw/t84ppn08D4NVxr57U+9Tn55M/ezZln36GNS2Nrq/NJ3hoy0FarTVL9i9hzso57K/YzwVdL+DezHu9/hwDt+1cAm9eCeGd4PqPIKJz467l243H9CoFr908nDPTT75MhRC+cvgU6OYa7vKPDwtkZEbLRFNYUUd2frmRbFy9ms+zcimparpDPizQYvRo4o2eTEPSSQwP7NCJRqYrtAPtdFL61lvGoHttLbF33kHMLbdgOmzQfU/ZHmavnM2yA8tIi0hj7oVzGZHkRxPYshfBwmsgqhtc/yGEGZPztNbGY3o/3kx6XAgvXz+UlJiOWz5CiBOllCIuzEZcmI2z0lveyFjY0KNxJZtteeUs+jmPhauaJRqbhYyE0MaeTIZrmRTRMRKNJBQvq9m6jdyZM6let47gESNInPkgtm7dWrSpqq/ipQ0vMX/zfKxmKzMyZ3BV76va5YE4bvv5E3j7RmPg/boPIMTofdQ7nMz8aBP/XbGX83rF8/QJPqb3dObxmXfCL8WG2ogNtR3Rcy+qqCU737h0lp1vJJqvt+Tz1qr9jW1CbRYy4kNd4zRNYzWdI/3rviO3EopSaiTGjKtKpdS1wGDgH1rrPV6NrgMzBt2fp+jVVzGHhpI0+3EiJkxo8Y+vtebz3Z/z5Konya/K59L0S7l7yN1eLdGwOafs+A/Kehfe/RV0HgzXvANBxqyY4so6fvP6albsKubWs9OZMbanxx/TK8SpLibURkyojRFpLRNNcWVdY5JpWH6ztYC3VzclmhCr2Ug0zSYDZLgSjckH/xfd7aE8DwxUSg0E7sGorzUfONtbgfmTK/+1yVgZ5177iqXLyJ01i/r9+4mYPJn4GfceMei+tXgrs1fOZlXeKvrE9OGps59iUPwgD0fuAev+azy2N+VMuHoh2Iz7SLbllXPzvJ/IK6vl71cMZNIZ7f+YXuF/ZDq350SHWBmeFsPwwxJNSWWdkWSajdMs2VbAO80STXBDookPo7KoPxZbCRW1dkJt3r0o5e672103IE4AntFav+KaoSWasRcUkDd7DmX/+x/Wbt1ImT+PkGHDWrQ5VHuI59Y9x4KtCwi3hvPgmQ8yOWNy+xdxdMdPr8D/fg9p58CV/wWrMS6yaHMev1uwlmCbhYXTR3BGimefky2EaFtUiJVh3aIZ1q3lw71Kq1yJxjU+sz2/gqXZBVSUGxN/cg9Ve/3GYncTSrlS6gHgWuAXSikTIBfKXbTTSenb75D/1FPo6mpi77idmF/9qsWgu18VcXTHD8/BFw9Aj3EwZR4EBKK15oUlO/m/L7bQr1MEL10/xCuVXIUQxy8y2MrQ1GiGprZMNENfvgp7bSSpMb/0egzuJpQrgKuBm7XWua6y8k94L6yOo2bbNnJnPkT12rUEDx9O4syZ2NJaDrpvKNjA4yseJ6soy/dFHN2x9ClY/DD0vhQuewUsVmrqHTzw3kbeX3uAiwck8cTlAwk6jrLrQgjfMJnrsAbnY2mHm4vd7qFgDMI7lFI9gF7Am94Ly/85a2qMQfd//9sYdH/8cSImthx0L6wu5B9r/sEH2z8gLijO90UcgdS6e9veqTV88yh89wT0nwoTnwezhfyyGqa/tpp1+0q554Ie3H7u8T+mVwhx6nM3oXwHjFZKRQFfYpRFuQK4xluB+bOKZcuNQfd9+4iYNIn4P8xoMehe76xn4ZaFPLvuWWocNUzrN41fD/j1cRVxbHdaw5d/hh+egcHXw8VPg8nMxv3GY3rLaup54dohjY9qFUKIw7mbUJTWuso1EP+c1vr/lFLrvRmYP7IXFhqD7p98gjU1lZT//IeQEcNbtFmZs5LHVz7eWMTxvqH30S2iWxvv2P4eLJrhWlvWtNHphM9mwE8vw7DpMG4OmEx8vP4gM95ZT0yIjXduPYs+nWQGjxCibW4nFKXUmRg9kobZXadPtT+tCSmvZ8f4i4xB99tvJ2Z6y0H33Mpcnlz1JF/s/sK/ijgei9MBH98Ja1+Hs+6ECx7GqeHpL7fyz6+3MzQ1iuevHUJsqB/UEBNC+DV3E8pdwAPA+1rrTUqpNOAb74XlX2JzqwmuchA4dACJsx7Cltb0rPGGIo4vb3wZp3Zy26DbuLHvjQRaAn0YsZscdvjgVtj4Npx9P4y5n8o6B79/ax1fbMpjamYyj0zseI/pFUL4hlsJRWu9BFiilApVSoW6nlFyp3dD8x9VoQFUhwTQa/68Fj2OJfuWMOenOewr3+d/RRyPxV4H794EP38M5z8Eo+5mf0kVt8xbxba8cv5ycR9uGpnq/z0sIYTfcLf0Sn+MO+OjjZeqALhea73Jm8H5i6ow45abhpPrnrI9zFk5h6UHlpIWkcZLF7zEmZ3O9GWIx0Vpp1HkMftLY7xkxK2s2m08prfO4eTVacM4u0ecr8MUQnjAUWd2epi7l7xeBH6vtf4GQCk1BpgLnOWluPxSVX0VczfOZd6meVjNVu7NvJere1/tX0Ucj0FpJyn23ZC92ZjJlTmNt37ax58+2EhyVDAv35BJelyor8M8rZzsYxGEd0gZmePnbkIJaUgmAFrrb5VSfjwH1rM0mnVdHPzfB5e0WxFHr3A6SLHvJkRXwqQXsfebymMfb+bfy3cxKiOWZ68eTERwx0mOQniTJPrj525C2amU+gvwmuv1tcBO74Tkf+adVUdWspPegTH+W8TRHRvfJlRXctDcmZAel3HHvFV8t62AG89K5c8X9W6XO2mFEKcudxPKTcAs4D3X66V0sMfunoy+B830yjXz53+86Z9FHN1hr4WvH6VaBZKl05n93HL2FlXx+OT+XDUsxdfRHTf561EI/+PuLK8STqNZXYcbutv4MXXYZAJG5eBDe1muRnB31Q3YVD1v3DL8iNLYQghxoo6aUJRSHwO6rf1a60s9HpHwvJpD8N0T1KT8gt9uu5kkUwmv3zaeLtHymF4hhOccq4fyZLtE4ec6/CNav/8XVBfznPlaNIqHgt+mS/Q0X0clTkFyKfL0dtSE4rqhUXRk5Xnww7McSruEf/0cymTrD8SbTuAxwEIIcQzu3ti4kSMvfR0CVgF/1VoXeTow4SFL5oCjjr9WX0Z4YABTLD/6OiIhxCnK3VlenwEO4L+u11cCwUAu8B/gEo9HJk5e0Q5YM4+cjCt5e4OVP43P4O8/PwLAQh+HJoRoHwt/3X5VPNxNKOdrrQc3e71RKbVGaz1YKXWtNwITHvD1I2izlRkFY+kcGcR1Z3Zl0c95vo5KCHGKcvdONrNSaljDC6XUUKBhDq3d41GJk3dgDWx6n21pN7Asx8y9Y3sQGNCBpz0LIfyeuz2UW4B/K6VCAQWUATe7yq887q3gxElY9BA6OIY7946iT1I4EwZ29nVEQohTnFs9FK31T1rr/sAgYKDWeoBrW6XW+q3j/VClVLRS6iulVLZrGdVGuxSl1JdKqZ+VUpuVUqmu7ecqpdYopbKUUvOUUu4mxtPDjq9h1xJWJN/E1hLF/b/shckkZeiFEN7lVkJRSkUopf4GLAYWK6WeUkpFnMTn3g8s1lp3d73n/W20mw88obXuDQwD8pVSJmAecKXWuh+wB7jhJGI5tTid8NVMnBEp3Jl9BqO7x/ILKUUvhGgH7o6h/BsoB6a6vsqAk7mDaQJGUsC1nHh4A6VUH8Citf4KQGtdobWuAmKAOq31NlfTr4DLTiKWU8um9yB3A5/G3kR+Ndw3rpevIxJCnCbcvVSUrrVuftKepZRadxKfm6C1znGt5wIJrbTpAZQqpd4DugGLMHoyhYBFKZWptV4FXA50OYlYTh32Ovj6Eepj+3Dv1h5MHNSJfp1PpiMphBDuc7eHUq2UGtXwQik1Eqg+2gFKqUWuMY7DvyY0b6e11rReL8wCjAbuBYYCacCNrvZXAn9XSq3E6Dk5jhLHdKXUKqXUqoKCAve+245qzTwo2c38kBtxahP3XNjT1xEJIU4j7vZQfgPMc42bKKAYuPFoB2itz29rn1IqTymVpLXOUUolAfmtNNsPrHM9vx6l1AfACOAVrfUPGMkGpdSFGL2ZtuJ4CXgJIDMzs81Clx1ebQUsmUNVpxE8uq0zN43sKsUfhRDtyt1ZXuu01gOBAUB/rfUZWuv1J/G5H9E0kH4D8GErbX4CIpVSDSPK5wKbAZRS8a6lDbgPeOEkYjk1/PAsVBbwN30NIbYAbjsnw9cRCSFOM8cqX//7NrYDoLX+2wl+7mzgLaXUzRiztKa63jcTuFVrfYvW2qGUuhdjVpkCVmM8xx5ghlLqYoyE+LzW+usTjOPUUFEA3/+TopSxvLwthvvGZRAVYvV1VEKI08yxLnmFeeNDXcUkz2tl+yqMmygbXn+F0Ss6vN0MYIY3YmuN35fkXvokur6av5RfRlJEINNGpvo6IiHEaehY5etntVcg4gQV74KfXmFf18l8uiWUJy6XEitCCN847jvMG4pCeiMYcQK+eQxtsnB33jh6JYYxeXDyUZu3Z+VRIcTpxd1pw81JDQ9/kbMBNr5FVperWF0SxH2/7IVZSqwIIXzkRBLK/zwehTgxi2ehAyO5fc/ZnJkWwxgpsSKE8KHjTiha6z97IxBxnHZ9B9sX8V3C9eypCuCB8b0aZ98JIYQvuPsI4HLafgTwPQ03H4p2ojV8NRNHWGfu3DmUiwckMSA50tdRCSFOc+4Oyj+Ncef6fzHGUK4E0oE1GIUjx3gjONGGzR/CwTW8n/wAVcUWZoyVEitCCN9z95LXpVrrF7XW5VrrMlc5k7Fa64VAq88yEV7iqIevH6E2uicP7OzHNcO70jUmxNdRCSGE2wmlSik1VSllcn1NBWpc+07d+lj+aO1rULSduQHXYgsI4I5zpcSKEMI/uJtQrgGuwyjimOdav1YpFQTc7qXYxOHqKuHbOZTHZ/LknjR+MyadmFCbr6MSQgjAzTEU16D7JW3sXua5cMRR/fg8VOTyuOUeEsIDuWlkN19HJIQQjdx9BHAPpdRipVSW6/UApZRMH25PVcWw/B/kJZ3Lf3M7c/f5PQiySokVIYT/cPeS11zgAaAeQGu9AWOml2gvS59C11XwQOkkuseHcvmQo5dYEUKI9uZuQgnWWq88bJvd08GINpTuhZUvsaPTpXxdEsN943phMZ9IkQMhhPAed89KhUqpdFwzupRSlwM5Rz9EeMw3j6NR/C73lwxLjea83vG+jkgIIY7g7o2Nt2E8RreXUuoAsAtj5pfwtrxNsP5NVne+hk07wnjvBimxIoTwT+4mlAPAq8A3QDRQhvHo3oe9FJdosPhhnLYw7th3DuP7JzI4Re4jFUL4J3cveX2IMW24HjgIVACV3gpKuOz5HrZ9zqLoqymwBzNjbC9fRySEEG1yt4eSrLUe59VIREuuApD2kETu3jOCq4al0C1WSqwIIfyXuz2U75VS/b0aiWhp66ewfyULgq8GSxB3ntfd1xEJIcRRudtDGQXcqJTaBdRiVBzWWusBXovsdOaww6JZ1ESkMXPfGdx5fjpxYVJiRQjh39xNKL/0ahSipfVvQuFWnon8M1GhwdwyWkqsCCH8n7u1vPZ4OxDhUl8N3zxGafRAnjnYm79O7E6Izd28L4QQviO3W/ublS9B+UEerrmCtLhQrhjaxdcRCSGEWySh+JPqElj6FAfjRvFecSp/GNuLACmxIoToIORaij9Z9jS6powZ9kkM6RrF2L4Jvo5ICCHcJn/++otDB2DFC2yNH8fyiiQe+KWUWBFCdCw+SShKqWil1FdKqWzX8oh6Ikqpc5RS65p91SilJrr2dVNKrVBKbVdKLVRKWdv/u/CwJbPR2smdeRdxYZ8EMlOjfR2REEIcF1/1UO4HFmutuwOLXa9b0Fp/o7UepLUeBJwLVAFfunbPAf6utc4ASoCb2ydsLynYCmtfZ0X0RHbUx/CHcVJiRQjR8fgqoUwA5rnW5wETj9H+cuAzrXWVMq4DnQu8cxzH+7fFD+O0BPO7g+cxNbMLGfGhvo5ICCGOm68SSoLWuuF5KrnAsUafrwTedK3HAKVa64YHfO0HOrd1oFJqulJqlVJqVUFBwcnE7B37VsKWT/g0fAplpkjuPl9KrAghOiavzfJSSi0CElvZ9afmL7TWWimlj/I+SUB/4IsTiUNr/RLGs1zIzMxs83N8wlUAsj4ojj8cGM2vzu1GfHigr6MSQogT4rWEorU+v619Sqk8pVSS1jrHlTDyj/JWU4H3tdb1rtdFQKRSyuLqpSRjPK+l48n+EvZ+z/yI2wgKCWf62em+jkgIIU6Yr+5D+QjjAV2zXcsPj9L2KuCBhheuHs03GOMqC9w43j85HbBoFlWhXXk8bzgPTuhOqJRYEcJr6uvr2b9/PzU1Nb4OxW8FBgaSnJxMQEDACR3vqzPYbOAtpdTNwB6MXghKqUzgVq31La7XqUAXYMlhx98HLFBK/RVYC7zSPmF70Ia3IH8TTwf9geSYcK4aluLriIQ4pe3fv5+wsDBSU1PlHq9WaK0pKipi//79dOt2YgVpfZJQtNZFwHmtbF8F3NLs9W5aGXDXWu8EhnkxRO+qr4FvHqU4og9z8wbwzNVSYkUIb6upqZFkchRKKWJiYjiZyUtyFvOFVa/AoX08VDWFAV2iGd+/tbkLQghPO95kcsWLP3DFiz94KRr/c7LJVhJKe6s5BN89yb6oEXxU3lNKrAhxGgkNbbrHbNy4cURGRnLxxRe32va2225j0KBB9OnTh6CgIAYNGsSgQYN45513Wm3fljVr1vD555+fVNzuklHg9rb8n1BdzD3VEzmvVzwj0mJ8HZEQwgdmzJhBVVUVL774Yqv7n332WQB2797NxRdfzLp1607oc9asWUNWVhbjxo074VjdJT2U9lSeCz8+R1b0BayqTeG+X0qJFSFOV+eddx5hYWEndGx2djZjx45lyJAh/OIXv2Dbtm0ALFiwgH79+jFw4EDOOeccqqurefjhh3njjTdOqHdzvKSH0p6WzEE76rgr7yKmDOlCj4QT+2USQpycWR9vYvPBsmO225xjtHFnHKVPp3BmXtL3pGNzx/Tp03n55ZdJT09n+fLl3H777Xz55ZfMmjWLb7/9loSEBEpLSwkKCuLBBx8kKyuLp59+2utxSUJpL4XbYfU8lkVcwv6iRF6/oIevIxJCdEClpaX8+OOPXHbZZY3b7HajEtXIkSO5/vrrmTJlCpMnT2732CShtJevH8FhtnF37lhuGtONxAgpsSKEr7jbk2jomSz89ZneDOe4aK2JjY1tdUxl7ty5rFixgk8++YTBgwezdu3ado1NxlDaw4HVsPkDPgyajCM4llvHSIkVIcSJiYqKIikpiffffx8Ap9PJ+vXrAdi5cycjRozgkUceISoqigMHDhAWFkZ5eXm7xCYJxdu0hkUPUWeL4sGCMdxxbnfCA0+srIEQ4tQxevRopkyZwuLFi0lOTuaLL9yvf7tgwQJeeOEFBg4cSN++ffnkk08AuPvuu+nfvz/9+/fnnHPOoV+/fpx77rmsX7+eM844QwblO7wdX8Ou73glaDpR0TFcM0JKrAhxuqqoqGhcX7p0qVvHpKamkpWV1WJbWlpaqwnoo48+OmJbXFwcq1atOs5IT4wkFG9yOmHRTCqDO/P34pE8cWVPbBazr6MSQrjJn8ZOOgK55OVNm96D3I08WT+Fnp1juWRAJ19HJIQQXiM9FG+x18HXj1AU2oP/FGbyxhW9MJmkxIoQ4tQlPRRvWf0fKNnNnysu5xc9EjgrI9bXEQkhhFdJD8UbasthyRz2hA3m88K+fColVoQQpwHpoXjDD89CVSH3lExm8hld6J0U7uuIhBAn4tWLjC/hFkkonlZRAN//i/VhZ7OBDH5/oZRYEUIY2rt8/fvvv88TTzxx0nG7Sy55edp3T6Drq/l9xSVMG5VK58ggX0ckhPBDnipfb7fbsVhaP5VPmjTJM8G6SXoonlS8C1b9m29DxlFo68pvx2T4OiIhhJ86mfL1o0aN4u677yYzM5NnnnmGDz/8kOHDh3PGGWdw4YUXkp+fD8DLL7/MXXfdBcC1117L7373O8466yzS0tIaS7d4kvRQPOmbR3EoM/cVjuf28RlEBEuJFSH80mf3Q+7GY7fL3WAs3RlHSewPv5x9cnEdB4fD0XgHfElJCZdeeilKKV544QWeeuop5syZc8Qx+fn5LF++nI0bNzJ16lSP92AkoXhKznrY+DbvBk0hIKAT153Z1dcRCSFOYVdccUXj+t69e5k6dSq5ubnU1tbSo0frY7cTJ05EKcWAAQM4cOCAx2OShOIpi2ZRFxDBX0suZNYVPQgMkBIrQvgtd3sSDT2Taf/zXiwnKCQkpHH9tttu449//CPjx49n0aJFzJ7d+vdns9ka17XWHo9JxlA8YecS2LGYl5hEclISEwZ29nVEQojTyKFDh+jcuTNaa+bNm+ezOCShnCytYdFMKmyJ/Kt8DPf/UkqsCCGO7WTK1x/uoYceYtKkSQwdOpSEhAQPRnl8lDe6Pf4qMzNTe7yM86b34e0beVD9lp2dJ/L6LcM9+/5CCI/4+eef6d279/Ed5MeXvLyltZ+TUmq11jrzWMfKGMrJcNTD4kcoCErj9ZKz+EhKrAhxajmNEoknyCWvk7FmPhTv4C8Vl3HpoGT6dY7wdURCCOEzPkkoSqlopdRXSqls1zKqlTbnKKXWNfuqUUpNdO27XSm1XSmllVJeL+N7xYs/cMWLP7TcWFcJS+awK3gAXzsHc8+FPb0dhhBC+DVf9VDuBxZrrbsDi12vW9Baf6O1HqS1HgScC1QBX7p2LwfOB/a0U7xH+vE5qMhjRulkrj8zlS7RwT4LRQgh/IGvEsoEoGFu2zxg4jHaXw58prWuAtBar9Va7/ZeeMdQWQTL/8naoLPYau3DbedIiRUhhPBVQknQWue41nOBY81zuxJ480Q+SCk1XSm1Sim1qqCg4ETe4khLn0LXVTCjdCK/HZNBVIjVM+8rhPAr0z6fxrTPp/k6jA7DawlFKbVIKZXVyteE5u20MW+5zbnLSqkkoD9wQpO0tdYvaa0ztdaZcXFxJ/IWLZXuRf80l0W286kMz2DayNSTf08hxGmhoXz9unXrOPPMM+nbty8DBgxg4cKFR7T1RPl6gDVr1vD55597JP5j8dq0Ya31+W3tU0rlKaWStNY5roSRf5S3mgq8r7Wu93iQbnqwaIZrbRl88xhODQ+WXsLvL5cSK0KI4xccHMz8+fPp3r07Bw8eZMiQIYwdO5bIyMjGNu6Wrz+WNWvWkJWVxbhx4zwS+9H46pLXR8ANrvUbgA+P0vYqTvByl8flbUKvX8Bb5vFEJKYyeXCyryMSQnRAPXr0oHv37gB06tSJ+Ph4jueSfHZ2NmPHjmXIkCH84he/YNu2bQAsWLCAfv36MXDgQM455xyqq6t5+OGHeeONN06od3O8fHVj42zgLaXUzRgztaYCKKUygVu11re4XqcCXYAlzQ9WSt0J/AFIBDYopT5tOMarFs2izhLK7PLxPH1ZL8xSYkWIDmnOyjlsKd5yzHYNbdwZR+kV3Yv7ht133LGsXLmSuro60tPT3T5m+vTpvPzyy6Snp7N8+XJuv/12vvzyS2bNmsW3335LQkICpaWlBAUF8eCDD5KVlYDaZh0AAAnqSURBVMXTTz993LEdL58kFK11EXBeK9tXAbc0e70bOKLSotb6n8A/vRjiEYKdlZD9BS+qa+iT1pUxPT0wHiOEOK3l5ORw3XXXMW/ePEwm9y4YlZaW8uOPP3LZZZc1brPb7QCMHDmS66+/nilTpjB58mSvxHw0UnrFHVqT4MihwhrHc2Xn89b4XiglvRMhOip3exINPZNXx73q8RjKysq46KKLePTRRxkxYoTbx2mtiY2NbXVMZe7cuaxYsYJPPvmEwYMHs3btWk+GfExSesUNYbqMYF3NnJpJnD8glQHJkcc+SAgh2lBXV8ekSZO4/vrrufzyy4/r2KioqP9v7/5jqyrvOI6/P2KlC7DM1YIMjJRNN+SXZNQfkZjBnBpmkO4HdbFbtmXJTFzmTDZncJkokUgIy5IlCy5owGFkOMQRdIky2WBEJ2qsVKsO2USIFeyU0UzpKN/9cU+7ioXe3p7bc3vv55U0OefkPLffpy33y3Oe534fxo8f37OF7/Hjx2lubgZg7969XHLJJSxdupQzzzyTAwcOMGbMGI4cOZJ6H/rihJKHmq53eCs+yUNdl/OTq1xixcwGZ8OGDWzfvp01a9b0LAceyCqu9evXs2rVKmbOnMnUqVPZsmULADfffDPTp09n+vTpzJ07l2nTpjFv3jyam5uZNWtW2U7KDytPaRYrO6/huksnc27NqP4bmJn1oaOjA4CmpiaampryajNp0iRaWlo+dG3y5Ml97p+yefPmj1yrra0l9W07TsIJJQ/3dl7BP+Ns1s1ziRWzSlKMuZNy5oSShyurmrn49L9TM3roV02YmQ0XTih5uKjq9axDMDMreZ6UNzOzVDihmJlZKvzIKw931qwA4KP1QM2snL3xzW8BcO5v7884kuHBIxQzsyEy1OXrN23axIoVK1KLvz8eoZiZDbE0y9cfO3aM00/v+628oaEh/eBPwQnFzGyInX/++T3HvcvX904opzJnzhzq6+vZsWMHTU1N1NXVsWzZMjo7O6mtrWXdunWMHTuW1atX91Qabmpqoqamhl27dtHW1sbKlStTTzhOKGZWcdqWLeNoa//l6z94JXdP91zKqYyc8jnOXrx4wLEUUr4eoKurq+cT8O+++y4LFixAEqtWrWLlypUsX778I20OHjzIzp072b17N4sWLXJCMTMrF4WUr+/W2NjYc7xv3z4WLVpEW1sbR48e/dAIqLeFCxciiRkzZnDgwIFBxd4XJxQzqzj5jiSKucqr0PL13UaN+n9dwRtvvJHFixczf/58tm7dyt13391nm5EjR/YcR8TAg+6HV3mZmQ2xwZSv78vhw4eZMGECEcHatWtTiLAwTihmZkNssOXrT7RkyRIaGhqor69n3LhxKUY6MCrGsKdUzZ49Owop49x4z1MA/O77l6YdkpkNkdbWVqZMmTKgNpX4wca+fk6SnouI2f219RxKHpxIzCpTJSWSNPiRl5mZpcIJxczMUuGEYmYVo5LmjAsx2J+PE4qZVYTq6mra29udVE4iImhvb6e6urrg1/CkvJlVhIkTJ7J//34OHTqUdSglq7q6mokTJxbc3gnFzCpCVVUVdXV1WYdR1vzIy8zMUuGEYmZmqXBCMTOzVFRU6RVJh4A3Cmx+FvBOiuFkqVz6Ui79APelVJVLXwbbj3Mjora/myoqoQyGpGfzqWUzHJRLX8qlH+C+lKpy6ctQ9cOPvMzMLBVOKGZmlgonlPz9JusAUlQufSmXfoD7UqrKpS9D0g/PoZiZWSo8QjEzs1Q4oQyApKWSXpT0gqTHJX0q65gKJWmFpFeS/myS9ImsYyqEpK9LeknScUnDcjWOpKslvSppj6Rbs46nUJLuk3RQUkvWsQyGpHMkbZP0cvK3dVPWMRVKUrWkZyQ1J325o6jfz4+88ifp4xHx7+T4h8AFEXFDxmEVRNKVwJMRcUzScoCI+GnGYQ2YpCnAceAe4McRMfA9njMkaQTwGvAlYD+wC/hGRLycaWAFkHQ50AHcHxHTso6nUJLGA+Mj4nlJY4DngIXD9HciYFREdEiqAv4K3BQRTxfj+3mEMgDdySQxChi22TgiHo+IY8np00DhJUYzFBGtEfFq1nEMwkXAnojYGxGdwHrg2oxjKkhEbAf+lXUcgxURb0XE88nxEaAVmJBtVIWJnI7ktCr5Ktr7lhPKAEm6S9KbwPXAz7OOJyXfBf6YdRAVagLwZq/z/QzTN69yJGkSMAv4W7aRFE7SCEkvAAeBJyKiaH1xQjmBpK2SWvr4uhYgIm6LiHOAB4AfZBvtqfXXl+Se24Bj5PpTkvLph1naJI0GNgI/OuHpxLASEV0RcSG5pxAXSSra40jvh3KCiLgiz1sfAB4Dbi9iOIPSX18kfRu4BvhilPBk2gB+J8PRAeCcXucTk2uWoWS+YSPwQEQ8nHU8aYiI9yRtA64GirJwwiOUAZB0Xq/Ta4FXsoplsCRdDdwCLIiI/2QdTwXbBZwnqU7SGcB1wOaMY6poyUT2vUBrRPwi63gGQ1Jt9wpOSR8jt/ijaO9bXuU1AJI2Ap8lt6roDeCGiBiW/5uUtAcYCbQnl54ejivWJDUAvwJqgfeAFyLiqmyjGhhJ84FfAiOA+yLiroxDKoikB4EvkKts+zZwe0Tcm2lQBZA0B9gB7Cb3bx1gcUQ8ll1UhZE0A1hL7m/rNGBDRNxZtO/nhGJmZmnwIy8zM0uFE4qZmaXCCcXMzFLhhGJmZqlwQjEzs1Q4oZilSFJH/3edsv3vJU1OjkdLukfS65Kek/RnSRdLOkPSdkn+YLKVFCcUsxIhaSowIiL2JpdWkyu2eF5EfB74DnBWUkTyT0BjNpGa9c0JxawIlLMiqTm2W1Jjcv00Sb9O9qJ5QtJjkr6WNLse+ENy36eBi4GfRcRxgIj4R0Q8mtz7SHK/WcnwkNmsOL4CXAjMJPfJ8V2StgOXAZOAC4Cx5Eqj35e0uQx4MDmeSu5T/10nef0WoL4okZsVyCMUs+KYAzyYVHp9G/gLuQQwB3goIo5HRBuwrVeb8cChfF48STSdyQZQZiXBCcWsdLwPVCfHLwEzkx0dT2Yk8EHRozLLkxOKWXHsABqTzY1qgcuBZ4CdwFeTuZRx5IopdmsFPgMQEa8DzwJ3JNVvkTRJ0peT4xrgnYj471B1yKw/TihmxbEJeBFoBp4EbkkecW0ktyvjy8A64HngcNLmUT6cYL4HjAP2SGoB1pDbdQ9gbnK/WclwtWGzISZpdER0JKOMZ4DLIqIt2a9iW3J+ssn47td4GLg1Il4bgpDN8uJVXmZDb0uy6dEZwNJk5EJEvC/pdnJ7yu87WeNkI65HnEys1HiEYmZmqfAcipmZpcIJxczMUuGEYmZmqXBCMTOzVDihmJlZKpxQzMwsFf8DtuIAwEWJKUcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x120971978>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    plt.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    plt.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "plt.legend()\n",
    "plt.xlabel( 'log(C)' )                                                                                                      \n",
    "plt.ylabel( 'neg-logloss' )\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/sklearn/linear_model/base.py:340: RuntimeWarning: overflow encountered in exp\n",
      "  np.exp(prob, prob)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[1, 10, 100, 1000], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l1',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#logisticREgressionCV L1\n",
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "Cs = [1, 10, 100, 1000]\n",
    "lrcv_L1 = LogisticRegressionCV(Cs=Cs, cv=5, scoring='neg_log_loss', penalty='l1', solver='liblinear', multi_class='ovr')\n",
    "lrcv_L1.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: array([[-0.22798591, -0.23059543, -0.233499  , -0.23575412],\n",
       "        [-0.22706685, -0.23143661, -0.23593124, -0.23818383],\n",
       "        [-0.22791823, -0.23022068, -0.23260365, -0.23352899],\n",
       "        [-0.2168826 , -0.22035763, -0.2234624 , -0.22552463],\n",
       "        [-0.22684719, -0.22912617, -0.23204336, -0.23449558]]),\n",
       " 1: array([[-0.48962691, -0.49129941, -0.4920169 , -0.49265104],\n",
       "        [-0.48277172, -0.4861471 , -0.48827839, -0.48935906],\n",
       "        [-0.48122676, -0.48370153, -0.48424917, -0.48430866],\n",
       "        [-0.48382958, -0.48597998, -0.4880755 , -0.48933958],\n",
       "        [-0.48495358, -0.48750063, -0.48939173, -0.49023888]]),\n",
       " 2: array([[-0.54658883, -0.54614615, -0.54984622, -0.54789298],\n",
       "        [-0.5158047 , -0.51693583, -0.5167166 , -0.51830073],\n",
       "        [-0.54423984, -0.54745524, -0.54515976, -0.54653862],\n",
       "        [-0.54434667, -0.54088301, -0.54871062, -0.54272634],\n",
       "        [-0.54103949, -0.54216746, -0.54427971, -0.54402561]])}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ -9.14961368e-01,   0.00000000e+00,   1.87197501e-02,\n",
       "         -1.43804701e-02,   6.15918177e-04,  -4.13541240e-03,\n",
       "         -7.93279651e-05,  -8.97871027e-01,  -3.15755989e-01,\n",
       "          3.23311556e-04,   2.12318876e-02,  -1.08349466e-02,\n",
       "          3.03257260e-02,  -1.32532364e-03,   6.79144156e-02,\n",
       "          0.00000000e+00,   7.60318668e-02,   2.21348768e-01,\n",
       "          1.55541195e-02,  -3.15098584e-01,  -2.53044994e-01,\n",
       "         -1.93739252e-01,  -3.36003325e-02,   0.00000000e+00,\n",
       "         -1.09384864e+00,   0.00000000e+00,  -1.51041856e+00,\n",
       "          0.00000000e+00,  -2.01473857e+00,  -9.45384814e-02,\n",
       "          8.52660725e-02,  -1.53491899e+00,   1.44045471e-01,\n",
       "         -2.75538542e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "         -8.32974481e-01,   0.00000000e+00,  -1.56011252e-01,\n",
       "          1.26868126e-01,   2.42054798e-01,  -2.43160079e+00,\n",
       "          0.00000000e+00,  -3.99685126e-01,  -7.41450749e-02,\n",
       "          5.55136576e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -1.87893819e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          1.84915282e-01,   0.00000000e+00,   1.08733711e+00,\n",
       "         -1.70667791e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "         -5.63845365e-02,  -1.15917275e-01,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   3.02645421e-02,\n",
       "          0.00000000e+00,   1.33605100e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -9.82004241e-02,\n",
       "          1.02040178e+00,   0.00000000e+00,  -2.88134221e-02,\n",
       "          0.00000000e+00,  -1.18639429e-01,   0.00000000e+00,\n",
       "          0.00000000e+00,   1.30685992e-01,   1.13008137e-01,\n",
       "         -2.31883221e-01,  -1.36263780e-01,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   9.19652819e-02,   0.00000000e+00,\n",
       "          3.98046053e-01,   0.00000000e+00,  -9.04283785e-03,\n",
       "          6.82495494e-01,  -2.72676224e-01,   0.00000000e+00,\n",
       "         -8.18875217e-01,   0.00000000e+00,   9.36936459e-01,\n",
       "          3.68661548e-01,  -9.51295921e-01,   2.74851379e-01,\n",
       "          0.00000000e+00,   1.34452885e+00,  -5.00272748e-01,\n",
       "          6.92707218e-02,  -6.35183768e-01,  -7.09275228e-01,\n",
       "          0.00000000e+00,  -3.56054127e-01,  -7.12398590e-01,\n",
       "          1.35647478e+00,   0.00000000e+00,   1.63817742e-01,\n",
       "         -1.34382328e+00,  -2.68068365e-01,   4.77596964e-01,\n",
       "         -1.27318701e+00,   1.64100418e-01,   4.73441651e-01,\n",
       "          0.00000000e+00,  -7.46952159e-01,   0.00000000e+00,\n",
       "          5.43772307e-01,   2.87401893e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,   1.02406392e-01,  -4.79546380e-02,\n",
       "          1.38816942e+00,   1.26185651e-01,   0.00000000e+00,\n",
       "          1.60076513e-01,  -4.53236156e-01,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -4.16266493e-01,\n",
       "          0.00000000e+00,  -4.69027133e-01,   0.00000000e+00,\n",
       "          0.00000000e+00,  -9.86485217e-02,   2.93308589e-01,\n",
       "          0.00000000e+00,   0.00000000e+00,  -2.65082311e-01,\n",
       "          0.00000000e+00,   0.00000000e+00,   7.79078932e-01,\n",
       "          2.09037662e-01,   2.13550006e-01,  -3.24763738e-02,\n",
       "          0.00000000e+00,   1.38177850e-01,   1.39953979e+00,\n",
       "         -3.92790592e-01,  -9.35616090e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,  -1.17410661e-01,  -3.49152671e-01,\n",
       "         -3.00947827e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          1.80871757e-02,   6.72465457e-01,   2.04743501e-01,\n",
       "          3.61037959e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "         -2.34097454e-01,   0.00000000e+00,   5.69377502e-02,\n",
       "          1.54075491e+00,   6.53710593e-01,   0.00000000e+00,\n",
       "         -2.98516635e-01,   0.00000000e+00,   2.34499801e-01,\n",
       "          0.00000000e+00,  -1.12189233e+00,  -2.94118024e-01,\n",
       "         -1.04584048e+00,  -5.00076143e-01,   1.35958893e-01,\n",
       "          4.98342681e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   3.59244275e-01,   9.07010368e-02,\n",
       "          0.00000000e+00,   3.10114965e-01,   7.41895262e-01,\n",
       "         -6.13054972e-01,  -1.35868737e+00,  -4.61791819e-01,\n",
       "          3.91074600e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "          1.01834635e+00,   5.77471315e-02,   3.37329162e-01,\n",
       "          0.00000000e+00,  -1.78412468e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   2.03728971e-01,  -4.78875250e-01,\n",
       "         -4.97166561e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -1.08882361e-03,   7.90917760e-01,\n",
       "         -4.64932201e-01,  -3.13681275e-01,  -4.72502418e-01,\n",
       "          0.00000000e+00,   2.90301161e-01,   0.00000000e+00,\n",
       "         -2.03268083e+00,   0.00000000e+00],\n",
       "       [ -4.13093315e-01,  -1.08227644e-02,   1.18714140e-02,\n",
       "         -7.66058519e-03,   1.15895173e-04,  -1.56247739e-03,\n",
       "         -1.05407931e-04,  -4.39664046e-01,  -4.95569225e-02,\n",
       "         -2.15261188e-04,   1.72579588e-02,  -1.17948935e-03,\n",
       "          1.29053742e-02,  -1.60844671e-03,   4.10131997e-02,\n",
       "          0.00000000e+00,   7.33162726e-02,   4.07162823e-02,\n",
       "         -1.40355123e-01,  -2.73140229e-01,   9.72516136e-02,\n",
       "          0.00000000e+00,   4.46704906e-02,   1.41575978e-01,\n",
       "         -2.88943072e-02,   0.00000000e+00,   0.00000000e+00,\n",
       "         -4.28203692e-01,   0.00000000e+00,   6.78509386e-03,\n",
       "          0.00000000e+00,   0.00000000e+00,  -2.64260998e-02,\n",
       "          0.00000000e+00,   3.77825911e-01,   6.83899222e-01,\n",
       "         -6.51006129e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          2.41799143e-02,   9.71528677e-03,   0.00000000e+00,\n",
       "          2.07517244e-01,   0.00000000e+00,   3.86726719e-01,\n",
       "          0.00000000e+00,   1.62277994e+00,   1.28203389e-01,\n",
       "          0.00000000e+00,  -6.61736703e-01,   0.00000000e+00,\n",
       "          2.50113578e-01,  -1.49521852e-01,  -1.68224603e+00,\n",
       "          6.35523105e-02,   0.00000000e+00,   5.26636306e-02,\n",
       "          0.00000000e+00,   9.95320267e-01,  -1.21118686e-02,\n",
       "          0.00000000e+00,  -6.66350780e-01,   3.62058982e-01,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   8.51726699e-02,\n",
       "         -2.45482773e-01,  -4.47522420e-02,  -5.43693778e-03,\n",
       "         -1.21543896e-01,   1.96279989e-02,  -4.76382469e-02,\n",
       "          4.17397928e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          1.98219539e-02,   6.46057797e-01,   2.30692948e-01,\n",
       "          3.22533645e-02,  -4.44501517e-02,  -1.01777334e-01,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "         -3.93612471e-01,   4.54336737e-02,   0.00000000e+00,\n",
       "          9.77850053e-02,   0.00000000e+00,  -2.06124018e-01,\n",
       "          4.21793277e-01,   1.51497969e-02,  -8.87226597e-02,\n",
       "          0.00000000e+00,  -4.66661906e-01,   4.20279881e-01,\n",
       "          2.00317418e-01,  -1.84185879e-01,   0.00000000e+00,\n",
       "         -9.12735747e-02,   8.18156706e-01,  -9.05089764e-02,\n",
       "          0.00000000e+00,   3.56612350e-01,  -1.77870520e-01,\n",
       "         -1.03348364e-01,   0.00000000e+00,  -5.57808392e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   2.33028058e-01,\n",
       "         -3.75493805e-01,   0.00000000e+00,   5.42016413e-01,\n",
       "          0.00000000e+00,   0.00000000e+00,  -1.06185419e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -1.62963091e-01,\n",
       "         -4.28130473e-02,   5.07517175e-01,   1.74861155e-01,\n",
       "          0.00000000e+00,  -5.94347937e-01,   3.11174673e-02,\n",
       "         -6.76438064e-01,  -5.31724014e-01,   0.00000000e+00,\n",
       "          2.68400405e-01,  -6.19949467e-01,   9.62857708e-02,\n",
       "          0.00000000e+00,   8.64650906e-01,   0.00000000e+00,\n",
       "          5.33666526e-01,   5.23205047e-01,   0.00000000e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,   0.00000000e+00,\n",
       "          0.00000000e+00,  -3.22486928e-02,   1.73948990e-01,\n",
       "         -7.01548791e-01,  -9.06439871e-02,   0.00000000e+00,\n",
       "          0.00000000e+00,  -6.87690391e-02,  -2.75652410e-02,\n",
       "         -1.94622498e-01,   4.47709056e-02,   0.00000000e+00,\n",
       "         -3.44455468e-01,  -2.52022829e-01,   0.00000000e+00,\n",
       "          0.00000000e+00,  -4.61975501e-01,   1.30957606e-01,\n",
       "         -1.20777931e-01,   0.00000000e+00,   0.00000000e+00,\n",
       "          3.64784191e-02,   4.35944826e-01,  -6.29878414e-01,\n",
       "         -4.59943519e-01,   0.00000000e+00,   5.25386519e-02,\n",
       "          7.78145437e-01,   8.24768496e-01,  -6.19021029e-01,\n",
       "          6.91562136e-01,   1.96771208e-01,   0.00000000e+00,\n",
       "          2.30523719e-01,   0.00000000e+00,  -2.21266877e-01,\n",
       "          1.24733201e-01,  -1.11430178e+00,   1.78699604e-01,\n",
       "          6.30368799e-01,  -6.07788263e-01,   2.12938997e-01,\n",
       "         -1.31122330e+00,   0.00000000e+00,  -4.19142016e-02,\n",
       "         -8.91819457e-01,   0.00000000e+00,   4.38627322e-02,\n",
       "          0.00000000e+00,   0.00000000e+00,   9.12436496e-01,\n",
       "         -9.72533618e-01,  -4.55929488e-01,   4.66077851e-02,\n",
       "          0.00000000e+00,   2.71896082e-01,   0.00000000e+00,\n",
       "          0.00000000e+00,   1.01640353e-02,   0.00000000e+00,\n",
       "          1.35242124e-01,   3.10920495e-01,   5.17752070e-03,\n",
       "          0.00000000e+00,   0.00000000e+00,   1.40505846e+00,\n",
       "         -6.13750555e-02,  -4.99008060e-01,  -4.18575812e-01,\n",
       "          5.19793171e-01,  -3.02523224e-01,   0.00000000e+00,\n",
       "          2.67303876e-01,  -1.15445146e-01,  -3.93541376e-01,\n",
       "          0.00000000e+00,  -6.21232049e-02,  -5.59065644e-01,\n",
       "          4.66059947e-01,   7.36013999e-01],\n",
       "       [  1.42584836e-01,  -2.49050738e-02,   1.06498522e-03,\n",
       "         -5.48174447e-05,   4.00526335e-05,   1.09441191e-04,\n",
       "          1.89337803e-04,   9.74236801e-02,   4.86815638e-02,\n",
       "          2.12444468e-05,   8.66883708e-03,  -1.25205656e-03,\n",
       "         -1.20483619e-02,   5.21569419e-04,  -4.70419357e-02,\n",
       "          0.00000000e+00,  -1.42689922e-01,  -1.71831490e-01,\n",
       "         -9.79556838e-02,   2.54264918e-01,   6.51609894e-02,\n",
       "          1.72207567e-03,  -1.90119813e-01,   6.20965373e-01,\n",
       "          4.55274422e-01,  -2.32201876e-01,   2.96890250e-01,\n",
       "          4.02164361e-01,   1.92209940e-01,   1.37026114e-01,\n",
       "         -1.43001664e+00,   2.92171391e-01,  -6.76346841e-03,\n",
       "          2.91871273e-02,  -2.53237853e-01,  -3.12089503e-01,\n",
       "          6.56483196e-01,   3.25707426e-01,   3.82641637e-08,\n",
       "          1.07202843e-03,  -5.38087258e-02,   3.79961987e-01,\n",
       "         -6.86719264e-02,   1.78813111e-01,  -5.09264962e-02,\n",
       "         -1.36283624e-01,  -1.19699203e+00,  -7.52071496e-02,\n",
       "          1.95974862e-02,   7.31492755e-01,  -1.09658749e-02,\n",
       "          0.00000000e+00,   1.36878826e-01,   1.95233429e+00,\n",
       "         -1.41098143e-01,   1.69179877e-01,  -7.86140243e-01,\n",
       "          1.02313842e-01,  -5.40120470e-01,  -5.03816173e-02,\n",
       "          1.34558268e-01,   6.54303790e-01,  -2.08386561e-01,\n",
       "         -3.45753805e-03,   8.07227101e-02,  -2.43655775e-01,\n",
       "          1.28544438e-01,  -1.05997674e+00,  -9.00917008e-01,\n",
       "         -1.04752977e-02,   9.50324800e-03,  -1.33891135e-01,\n",
       "         -6.77930988e-02,  -1.18241562e-01,   1.16786601e-01,\n",
       "         -4.84580739e-01,  -1.40283333e-01,  -2.51652716e-02,\n",
       "         -1.58304968e-01,  -2.75997637e-01,  -1.00973378e-02,\n",
       "          2.97332212e-02,   2.93790421e-01,   9.18133742e-01,\n",
       "          4.18969900e-01,   7.52784550e-03,  -2.94354541e-01,\n",
       "          2.37594827e-01,   1.22216812e-01,   3.21633011e-02,\n",
       "         -2.71262628e-01,  -2.47266913e-01,   6.49111423e-01,\n",
       "         -5.08516454e-01,   3.68357057e-01,   1.19525781e-01,\n",
       "          2.13555980e-01,   2.36197335e-01,  -1.70488213e-01,\n",
       "         -6.04574158e-01,   1.44458649e-01,  -1.81119850e-01,\n",
       "          8.00221151e-02,  -9.99919763e-01,   3.19285595e-01,\n",
       "          1.03026581e-01,  -8.67449277e-01,   4.12251713e-01,\n",
       "          0.00000000e+00,   5.49882174e-01,  -3.16258545e-01,\n",
       "         -1.63431742e-02,   1.64426803e-02,   1.03854250e-01,\n",
       "          1.05258373e+00,  -9.25691366e-02,  -1.06085961e+00,\n",
       "          0.00000000e+00,   0.00000000e+00,  -2.63845771e-02,\n",
       "         -4.53010074e-02,   3.33564644e-02,  -1.80166893e-01,\n",
       "         -2.75937988e-02,  -1.02371436e+00,  -9.34430916e-02,\n",
       "          2.16102734e-01,   4.31534731e-01,   2.44645087e-01,\n",
       "          0.00000000e+00,   3.76845619e-01,   5.86221217e-02,\n",
       "         -2.11747862e-01,   2.21149261e-01,   1.27377063e-01,\n",
       "          9.90997001e-01,  -4.01948342e-01,   1.84199913e-01,\n",
       "         -3.89757286e-03,  -2.63835452e-01,   8.17755207e-02,\n",
       "         -1.64180309e-01,   3.85254955e-01,  -5.49863121e-02,\n",
       "          1.39589799e-02,  -1.48802521e-01,  -5.46707485e-01,\n",
       "          4.26596911e-01,  -2.23884213e-02,   1.76466172e-02,\n",
       "         -5.03232418e-01,  -1.00237941e-01,  -1.36320688e-01,\n",
       "         -1.23594002e-02,  -3.88658488e-01,  -1.19761461e-02,\n",
       "          6.51653598e-01,  -6.30400173e-03,   3.05940290e-01,\n",
       "          9.01598852e-02,   4.14710398e-01,  -1.05220349e-01,\n",
       "          1.29325004e-01,   0.00000000e+00,   2.89649132e-05,\n",
       "         -6.25730010e-01,  -4.12781038e-01,  -4.13029543e-02,\n",
       "          4.35870703e-01,   3.80597669e-02,   6.83843003e-02,\n",
       "         -2.22911538e-01,  -5.29771077e-01,   8.38169014e-02,\n",
       "         -2.14342861e+00,  -6.30148173e-01,   3.08975158e-01,\n",
       "          9.16912082e-03,  -1.76999925e-02,  -5.34127737e-02,\n",
       "         -1.30913186e+00,   1.28836900e-01,  -1.87760636e-01,\n",
       "          0.00000000e+00,   8.86491371e-01,  -7.30777893e-02,\n",
       "          1.47532504e-01,  -1.52759245e-01,   5.70542069e-02,\n",
       "          6.32416319e-01,  -3.67526744e-02,  -3.74590906e-03,\n",
       "          0.00000000e+00,  -2.90380020e-02,  -1.34083674e+00,\n",
       "          9.68403259e-01,   5.43975217e-01,  -1.50287995e-01,\n",
       "         -4.91214717e-01,   3.55315176e-02,   1.68630061e-01,\n",
       "          2.39402359e-01,  -3.61036112e-02,  -2.21269784e-01,\n",
       "         -6.80025386e-01,  -6.97937367e-02,  -4.03365457e-02,\n",
       "         -7.30740705e-02,   6.79657056e-02,  -5.02702605e-01,\n",
       "          5.57079516e-01,   1.72455489e-01,   6.13442593e-01,\n",
       "         -6.12322445e-01,   2.16268997e-01,   1.81278617e-01,\n",
       "         -6.01421698e-02,   1.39944384e-01,   1.60062528e-01,\n",
       "          2.42581233e-02,  -1.52749384e-01,   3.72307537e-01,\n",
       "          0.00000000e+00,  -3.53235628e-01]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8VdW5//HPQyCMAQKEQaYwyqgMYVJ7tVYt1taheBUqIAqC+tNqq7XWam/Ve9uq91rnKgoOaAFnsWppa7VWLUPCDAICEiaBMIUxZHp+f5wdmlKSHJKc7JPk+369zsuz91n78CwP5Ju99zprmbsjIiJSXnXCLkBERKo3BYmIiFSIgkRERCpEQSIiIhWiIBERkQpRkIiISIUoSEREpEIUJCIiUiEKEhERqZC6YRdQFVq1auWpqalhlyEiUq1kZGTscveUstrFNEjMbCTwKJAAPOfuvymh3SjgdWCIu6eb2fnAb4BEIBf4ibv/NWj7P8B4INndm0RTR2pqKunp6RXuj4hIbWJmmdG0i9mlLTNLAJ4ELgT6AGPMrM8J2iUBtwDzi+3eBXzP3fsDVwMzir32LjA0VnWLiMjJieU9kqHAOnff4O65wCzgkhO0ux94AMgp2uHui919W7C5EmhoZvWD1+a5+9cxrFtERE5CLIOkPbC52PaWYN8xZjYI6Oju75XyPqOARe5+9GT+cDObbGbpZpaelZV1MoeKiMhJCG3UlpnVAR4GbiulTV8iZytTTvb93X2qu6e5e1pKSpn3ikREpJxiGSRbgY7FtjsE+4okAf2Aj81sIzAcmGNmaQBm1gF4Cxjv7utjWKeIiFRALINkIdDDzLqYWSIwGphT9KK7Z7t7K3dPdfdUYB5wcTBqqznwHnCnu38WwxpFRKSCYhYk7p4P3ATMBb4AXnX3lWZ2n5ldXMbhNwHdgV+Y2ZLg0RrAzB40sy1AIzPbYma/jFUfRESkbFYbltpNS0tzfY9ERGqLbfuOkJ65lzXb9/OTb/cq9/uYWYa7p5XVrlZ8s11EpKYqLHTW7jzAwo17Sd+4h/SNe9m67wgAjRMTmHhWV1o0ToxpDQoSEZFqJCevgKWb95GeuZeFG/eQkbmXAzn5ALROqs+Q1BZM+kYXhqS2oFfbJOomxH5wroJERCSO7TmUGznTyIyccSzfmk1eQeSWRI/WTfjuaacwJDWZIakt6JDcEDOr8hoVJCIiccLdydx9+FhoLNy4h/VZhwBITKjDaR2aMfGsrgxJTWZQp2SSY3zJKloKEhGRkOQVFPLF1/uP3d9YuHEvuw5GJvFo1rAeaZ2TGTW4A0NSW9C/fTMa1EsIueITU5CIiFSRg0fzWbxp77HgWLxpH0fyCgDokNyQb/RoRVpwmap7ShPq1Kn6y1TloSAREYmRHftzWBiMpErP3MOqbfspdKhj0LtdU64c0pG01GTSOregbbMGYZdbbgoSEZFKUFjorMs6GBlJtXEvCzP3sHlPZBhuw3oJDOzUnJvO7cGQ1GQGdGxOUoN6IVdceRQkIiLlkJNXwIqt2f/8/kbmXrKP5AHQqkl9hqQmM+GMLqR1TqbPKU2pVwXDcMOiIBERicK+w7lkZP7z/sayLdnkFhQC0C2lMSP7tj12f6Nzy0ahDMMNi4JEROQ47s6WvUdYGIykysjcw9odBwGol2D0a9+MCWemktY5mcGdk2nZpH7IFYdLQSIitV5+QSGrtx+I3BgPvsOxY39kGG5Sg7oM7pzMJQPak9Y5mdM7No/bYbhhUZCISK1zODefJZv2RS5TZe5hUeZeDuVGhuG2b96Q4V1bkpbagrTOyfRsk0RCNRmGGxYFiYjUeDsP5ERGUgXBsXLbfgoKHTPo1bYp3x/UITIMN7UF7Zs3DLvcakdBIiI1iruzPuvQsW+KZ2TuYePuwwDUr1uHAR2bc8PZ3UhLTWZQ52Sa1qBhuGFRkIhItZabX8jyrdn/MrHh3sORYbgtGieS1jmZq4Z1Ji01mb6nNCOxbs0dhhsWBYmIVCvZR/JYtOmfc1Mt3byPo/mRYbhdWjXmvN5tGJLagsGpyXRt1bhWDcMNi4JEROLa1n1Hjs2Em75xL2t2HMAd6tYx+rZvxrjhkbONwZ1bkJJUu4fhhkVBIiJxo6DQWbP9AOmZwf2NjXvYlp0DQJP6dRnYqTnf6d+OtGCakUaJ+hEWD/QpiEhojuQWsGTzvmP3NxZl7uXA0chqf22bNiAtNZkpqS1IS02mV9umGoYbpxQkIlJldh88WmzRpr2s2JpNfmFktb9T2yRx8YBTGBIER/vm4az2JydPQSIiMeHubNx9OLi3Ebm/sWFXsNpf3ToM6NCcyf/RNXJ/o1MLmjXSMNzqSkEiIpUir6CQldv2H7sxnpG5l10HcwFo3qgeaZ1bcMWQjgxJTaZf+2bUr6tpRmoKBYmIVMjBo/n89I1lfPjFDnLyIsNwO7VoxH/0TGFIaguGpCbTtVX1We1PTp6CRETKbc+hXCY8v4CV2/Zz1bBOkTmqOifTumn1Xe1PTp6CRETKZXt2DuOmzWfTnsM8M3Yw5/VpE3ZJEhIFiYictI27DnHVc/PJPpLHi9cOZXjXlmGXJCFSkIjISVm1bT/jpy+goLCQmdcNp3+HZmGXJCFTkIhI1DIy93DN8wtpXL8usyaPoHvrpLBLkjigIBGRqPxtbRbXz8igbbMGzJg4lA7JjcIuSeKEgkREyvTesq+5dfZierRO4sVrh2pyRPkXChIRKdWsBZu4663lDOqUzLQJQ2jWUN9Al38V0xVezGykma0xs3Vmdmcp7UaZmZtZWrB9vpllmNny4L/nFms7ONi/zsweM03GIxIzUz9Zz51vLucbPVKYMXGYQkROKGZBYmYJwJPAhUAfYIyZ9TlBuyTgFmB+sd27gO+5e3/gamBGsdd+B1wH9AgeI2PSAZFazN158I+r+dX7q/nuae14dnwaDRM1pYmcWCzPSIYC69x9g7vnArOAS07Q7n7gASCnaIe7L3b3bcHmSqChmdU3s3ZAU3ef5+4OvARcGsM+iNQ6BYXO3W+v4KmP1zNmaCceHT1Qy9NKqWL5t6M9sLnY9pZg3zFmNgjo6O7vlfI+o4BF7n40OH5Lae8pIuWXV1DIrbOX8Mr8TVx/djd+dVk/rQEiZQrtZruZ1QEeBiaU0qYvkbOVC8rx/pOByQCdOnUqX5EitciR3AJufCWDj9Zk8dORvbjhnG5hlyTVRCzPSLYCHYttdwj2FUkC+gEfm9lGYDgwp9gN9w7AW8B4d19f7D07lPKex7j7VHdPc/e0lJSUSuiOSM21PyePq6cv4OO1Wfz6+/0VInJSYhkkC4EeZtbFzBKB0cCcohfdPdvdW7l7qrunAvOAi9093cyaA+8Bd7r7Z8WO+RrYb2bDg9Fa44F3YtgHkRpv18GjjJk6j8Wb9/L4mIGMGaozeDk5MQsSd88HbgLmAl8Ar7r7SjO7z8wuLuPwm4DuwC/MbEnwaB28diPwHLAOWA98EJseiNR8W/cd4Yqn/8H6rIM8Oz6N7552StglSTVkkcFPNVtaWpqnp6eHXYZIXFmfdZBxz83nwNF8np8whLTUFmGXJHHGzDLcPa2sdvpmu0gttGJrNuOnL6COwazJw+l7imbwlfJTkIjUMvM37GbSi+k0bViPGROH0jWlSdglSTWnIBGpRf66egc3vLyIDskNeXnSMNo1axh2SVIDKEhEaol3lmzltleX0rtdU168digtGieGXZLUEAoSkVpgxrxMfvHOCoamtuC5q9NIaqDJF6XyKEhEajB356mP1/PQ3DWc17s1T/xgEA3qafJFqVwKEpEayt359QermfrJBi4b2J4HLz+NegmafFEqn4JEpAYqKHTuenM5s9M3M35EZ375vb7U0eSLEiMKEpEa5mh+AbfOWsIHK7bzw3O786Pze6L13ySWFCQiNcjh3HymzMjg71/u4u6LejPpG13DLklqAQWJSA2RfTiPa15YwJLN+3jw8tO4Iq1j2QeJVAIFiUgNsPNADuOnLWBD1iGeumoQI/u1C7skqUUUJCLV3OY9hxk7bT5ZB44yfcIQzurRKuySpJZRkIhUY2t3HGDctPnk5BXyyqRhDOyUHHZJUgspSESqqSWb9zHh+QUkJtTh1SkjOLVtUtglSS2lIBGphj5ft4vrXkqnRZNEXpk4nE4tG4VdktRiChKRauZPK7dz08zFdGnZmJcmDqVN0wZhlyS1nIJEpBp5I2MLd7yxjP7tm/HCNUNo3kgz+Er4FCQi1cTzn33Fve+u4szuLZk6Lo3G9fXPV+KD/iaKxDl359EPv+SRv3zJt/u24bExA6lfVzP4SvxQkIjEscJC574/rOKFzzdy+eAO/Ob7/amrGXwlzihIROJUfkEhd7yxjDcXbWXiWV34+Xd6awZfiUsKEpE4lJNXwM0zF/PnVTu47fye3HRud83gK3FLQSISZw4ezee6F9P5x4bd3HtxX64+IzXskkRKpSARiSN7D+Uy4fkFrNi2n0euHMClA9uHXZJImRQkInFie3YO46bNJ3PPYZ4ZO5jz+rQJuySRqChIROLAxl2HGDttPvsO5/HiNUMZ0a1l2CWJRE1BIhKyL77ez7hpCygoLOT31w3jtA7Nwy5J5KQoSERClJG5h2ueX0ijxLrMmjyC7q01g69UPwoSkZB8sjaLKTMyaNO0Pi9PGkaHZM3gK9WTgkQkBO8v/5pbZi2me+skXrp2KClJ9cMuSaTcFCQiVWz2wk387M3lDOqUzLQJQ2jWsF7YJYlUSFST9pjZmWbWOHg+1sweNrPOURw30szWmNk6M7uzlHajzMzNLC3YbmlmH5nZQTN74ri2V5rZMjNbaWYPRFO/SLyY+sl6fvrGcr7RI4WXJg5ViEiNEO3sb78DDpvZ6cBtwHrgpdIOMLME4EngQqAPMMbM+pygXRJwCzC/2O4c4B7g9uPatgQeAr7l7n2Btmb2rSj7IBIad+ehuav51furuei0djw7Po1GibogIDVDtEGS7+4OXAI84e5PAmUNLxkKrHP3De6eC8wKjj/e/cADRMIDAHc/5O6fFt8X6Ap86e5ZwfZfgFFR9kEkFIWFzj3vrODJj9YzZmgnHhs9kMS6msFXao5o/zYfMLOfAWOB98ysDlDWOXl7YHOx7S3BvmPMbBDQ0d3fi7KOdcCpZpZqZnWBS4GOUR4rUuXyCgq5dfYSXp63ievP7savLutHgmbwlRom2iC5EjgKTHT37UAHIpeYyi0Io4eJXCqLirvvBW4AZgN/BzYCBSW8/2QzSzez9KysrBM1EYmpI7kFTH4pnTlLt/HTkb2488JemsFXaqRoL9IeAB519wIz6wn0AmaWccxW/vVsoUOwr0gS0A/4OPjH1RaYY2YXu3t6SW/q7u8C70IkLCghSNx9KjAVIC0tzcuoVaRS7c/JY9IL6SzM3MOvLuvPD4Z1CrskkZiJ9ozkE6C+mbUH/gSMA14o45iFQA8z62JmicBoYE7Ri+6e7e6t3D3V3VOBeUCpIQJgZq2D/yYDNwLPRdkHkSqx6+BRxkydx6JNe3ls9ECFiNR40Z6RmLsfNrOJwFPu/qCZLS3tAHfPN7ObgLlAAjDd3Vea2X1AurvPKe14M9sINAUSzexS4AJ3XwU8GoweA7jP3ddG2QeRmNu67wjjnpvPtuwjPHt1Gt88tXXYJYnEXNRBYmYjgKuAicG+Ms9m3P194P3j9v2ihLbnHLedWkK7MWWXK1L11mcdZNxz8zlwNJ8ZE4cxJLVF2CWJVIlog+RW4GfAW8FZRVfgo9iVJVK9rNiazdXTF2AGsyYPp+8pzcIuSaTKRBUk7v434G9m1sTMmrj7BuCHsS1NpHqYv2E3k15Mp2nDesyYOJSuKU3CLkmkSkU7RUp/M1sMrARWmVmGmfWNbWki8e+j1TsZP30BrZvW57XrRyhEpFaKdtTWM8CP3b2zu3ci8t2PZ2NXlkj8e2fJVq57KZ2ebZJ4dcoITmneMOySREIR7T2Sxu5+7J6Iu39cNImjSG00Y14mv3hnBUNSWzDt6jSSGmjyRam9og2SDWZ2DzAj2B4LbIhNSSLxy9156uP1PDR3Dd/q1ZonrxpEg3oJYZclEqpoL21dC6QAbwaPlGCfSK3h7vzmg9U8NHcNlw44hafHDVaIiBD9qK29aJSW1GIFhc5dby5ndvpmxo/ozC+/15c6mnxRBCgjSMzsXaDEearc/eJKr0gkzhzNL+BHs5fw/vLt3Hxud358fk9NvihSTFlnJP9bJVWIxKnDuflMmZHB37/cxd0X9WbSN7qGXZJI3Ck1SIIvIorUStmH87jmhQUs2byPB0edxhVDtPSNyIlEdY/EzJbz75e4soF04L/dfXdlFyYSpp0Hchg/bQEbsg7x1FWDGNmvXdglicStaIf/fkBk3Y/fB9ujgUbAdiLTyX+v0isTCcnmPYcZO20+WQeOMm1CGt/okRJ2SSJxLdogOc/dBxXbXm5mi9x9kJmNjUVhImH4cscBxk6bT05eIS9PGsagTslhlyQS96L9HkmCmQ0t2jCzIUTWGAHIr/SqREKwdPM+/vOZf1DoMHvKcIWISJSiPSOZBEw3syaAAfuBicE0Kb+OVXEiVeXz9bu47sV0WjRJ5OWJw+jcUjMAiUQr2i8kLgT6m1mzYDu72MuvxqIwkaryp5XbuWnmYlJbNmLGxGG0adog7JJEqpVop5FvZmYPAx8CH5rZ/xWFikh19kbGFm54ZRG92zVl9uQRChGRcoj2Hsl04ABwRfDYDzwfq6JEqsLzn33Fba8tZXjXFvx+0jCSGyeGXZJItRTtPZJu7j6q2Pa9ZrYkFgWJxJq78+iHX/LIX77kgj5teGzMQE2+KFIB0Z6RHDGzs4o2zOxM4EhsShKJncJC5953V/HIX77k8sEdeErTwItUWLRnJDcALwb3RQzYA0yIVVEisZBfUMhP31jOG4u2cO2ZXbj7ot6awVekEkQ7amsJcLqZNQ2298e0KpFKlpNXwM0zF/PnVTv48fk9ufnc7prBV6SSlDWN/I9L2A+Auz8cg5pEKtXBo/lMfimdz9fv5pff68OEM7uEXZJIjVLWGUlSlVQhEiN7D+Uy4fkFrNi2n99eeTqXDewQdkkiNU5Z08jfW1WFiFS27dk5jJs2n8w9h3l67GDO79Mm7JJEaqRoR20dY2aLYlGISGXauOsQlz/9Odv2HeGFa4YoRERiKNpRW8XpDqXEtS++3s+4aQsoKCxk5uThnNahedglidRo5QmS9yq9CpFKkpG5l2ueX0CjxLrMmjyC7q11m08k1k46SNz97lgUIlJRn6zNYsqMDNo0rc+MicPo2KJR2CWJ1ArRLrV7gJKX2r3N3TdUdmEiJ+P95V9zy6zFdEtpwksTh9I6SZMvilSVaM9IHgG2EFlq14gstdsNWERkQsdzYlGcSDReXbiZO99cxsBOyUyfMIRmDeuFXZJIrRLtqK2L3f0Zdz/g7vvdfSrwbXefDWgZOQnNs59s4I43lnFWjxRmTByqEBEJQbRBctjMrjCzOsHjCiAneO34S17HmNlIM1tjZuvM7M5S2o0yMzeztGC7pZl9ZGYHzeyJ49qOMbPlZrbMzP5oZq2i7IPUIO7OQ3NX8z/vf8FF/dvx3Pg0GiWWZ+yIiFRUtEFyFTAO2AnsCJ6PNbOGwE0nOsDMEoAngQuBPsAYM+tzgnZJwC3A/GK7c4B7gNuPa1sXeBT4prufBiwr6c+Xmquw0LnnnRU8+dF6xgztyGNjBpJY96S/EiUilSTaSRs3AN8r4eVPS9g/FFhXdCPezGYBlwCrjmt3P/AA8JNif94h4FMz635cWwsejc1sN9AUWBdNH6RmyCso5PbXlvLOkm1MObsrd47spckXRUIW7VK7Pc3sQzNbEWyfZmZlDQNuD2wutr0l2Ff8fQcBHd09qu+muHsekSntlwPbiJzpTCuh5slmlm5m6VlZWdG8vcS5nftz+MGz83hnyTbuGHkqP7uwt0JEJA5Eez3gWeBnQB6Auy8jMnKr3MysDvAwcNtJHFOPSJAMBE4hcmnrZydq6+5T3T3N3dNSUlIqUqrEgYUb93DR45+yYut+Hh09gBvPOf5kVUTCEm2QNHL3Bcftyy/jmK1Ax2LbHYJ9RZKAfsDHZrYRGA7MKbrhXoIBAO6+3t0deBU4o+zypbpyd57/7CvGTJ1H48QE3vp/Z3DJgPZlHygiVSbaYS67zKwbwQgtM7sc+LqMYxYCPcysC5EAGQ38oOhFd88Gjo24MrOPgdvdPb2U99wK9DGzFHfPAs4HvoiyD1LNHM7N52dvLuedJds4r3cbHr7ydJo20PBekXgTbZD8P2Aq0MvMtgJfERnJVSJ3zzezm4C5QAIw3d1Xmtl9QLq7zynt+OAspSmQaGaXAhe4+yozuxf4xMzygEy05G+N9NWuQ1w/I4O1Ow/wk2+fyg1nd9OyuCJxyiJXiMpoZFYfuBxIBVoA+wF39/tiWl0lSUtL8/T00k50JJ78edUOfjx7CXUTjEdHD+Q/euoel0gYzCzD3Uu73QBEf0byDrCPyJQo2ypSmEhJCgqd3/55LU98tI7+7Zvxu7GD6JCsiRdF4l20QdLB3UfGtBKp1fYeyuWHsxbz9y93cWVaR+69pC8N6iWEXZaIRCHaIPnczPq7+/KYViO10vIt2Vz/cgZZB47ym+/3Z/TQTmGXJCInIdogOQuYYGZfAUeJfLvcg2lKRMpt9sJN3PPOSlKa1Oe160dweketZihS3UQbJBfGtAqpdXLyCrj33ZXMXLCZs7q34rExA2nRODHsskSkHKKdaysz1oVI7bF13xFueDmDZVuyufGcbtx2wakkaGivSLWlebelSn365S5unrmI/ALnmXGD+XbftmGXJCIVpCCRKuHu/O5v6/nfuWvo3roJT48dTNeUJmGXJSKVQEEiMbc/J4/bX13Kn1bt4LunteOBUafRuL7+6onUFPrXLDG1dscBrp+RQeaew9zz3T5ce2aqpn4XqWEUJBIz7y7dxh2vL6Nx/brMvG44Q7u0CLskEYkBBYlUuryCQn7zwWqmffoVaZ2TefKqQbRp2iDsskQkRhQkUql2HsjhplcWs2DjHiackcpd3+mt9dRFajgFiVSa9I17uPGVRezPyeORKwdw6UAtQCVSGyhIpMLcnRc/38h/v/cF7ZMb8uK1Q+ndrmnYZYlIFVGQSIUczs3nrjeX8/aSbZzXuzX/d8UAmjXUKoYitYmCRMpt465DXP9yBmt2HOD2C3py4zndtYqhSC2kIJFy+cuqHfzo1SUk1DFeuGYoZ2sVQ5FaS0EiJ6Wg0HnkL2t5/K/r6Ne+Kb+7ajAdW2gVQ5HaTEEiUdt7KJdbZi/hk7VZ/OfgDtx/aT+tYigiChKJzoqtkVUMd+4/yq+/35/RQzpqqhMRARQkEoVX0zdz99sraNU4kVevH8EArWIoIsUoSKRER/ML+OWcVcxcsIkzu7fksdEDadmkfthliUicUZDICW0LVjFcuiWbG87pxm3n96RugqY6EZF/pyCRf/PZul3cPHMxufmFPD12MCP7aRVDESmZgkSOcXee/tsGHpq7mm4pTXhmnFYxFJGyKUgEgAM5edz+2lLmrtQqhiJycvSTQvhyxwGmaBVDESknBUkt94dlkVUMGyXW5feThjGsa8uwSxKRakZBUkvlB6sYPvfpVwzunMxTWsVQRMpJQVILZR04yk2/X8T8r7SKoYhUnIKklsnIjKximH0kj99eeTqXDewQdkkiUs3F9NdQMxtpZmvMbJ2Z3VlKu1Fm5maWFmy3NLOPzOygmT1RrF2SmS0p9thlZo/Esg81hbvz0j82MnrqPBrUS+CtG89UiIhIpYjZGYmZJQBPAucDW4CFZjbH3Vcd1y4JuAWYX2x3DnAP0C94AODuB4ABxY7NAN6MVR9qiiO5Bdz11nLeWryVb/VqzcNXahVDEak8sTwjGQqsc/cN7p4LzAIuOUG7+4EHiIQHAO5+yN0/Lb7veGbWE2gN/L1Sq65hMncf4rKnPuPtJVu57fyePDs+TSEiIpUqlkHSHthcbHtLsO8YMxsEdHT398rx/qOB2e7u5S+xZvvwix189/FP+To7h+cnDOHmb/XQUrgiUulCu9luZnWAh4EJ5XyL0cC4Ut5/MjAZoFOnTuX8I6qngkLn0Q+/5LEPv6TvKU15eqxWMRSR2IllkGwFOhbb7hDsK5JE5P7Hx8G3qNsCc8zsYndPL+2Nzex0oK67Z5TUxt2nAlMB0tLSas1Zy77Dudwyawl/0yqGIlJFYhkkC4EeZtaFSICMBn5Q9KK7ZwOtirbN7GPg9rJCJDAGmFmp1dYAxVcx/NVl/RkzVKsYikjsxSxI3D3fzG4C5gIJwHR3X2lm9wHp7j6ntOPNbCPQFEg0s0uBC4qN+LoC+E6saq+OXs/Yws/fWk4LrWIoIlUspvdI3P194P3j9v2ihLbnHLedWsr7dq2E8mqEo/kF3PfuKl6Zv4kzurXk8TFaxVBEqpa+2V6NfZ19hBteXsSSzfu4/uxu3H6BVjEUkaqnIKmmPg9WMTyaX8jTYwcxsl+7sEsSkVpKQVLNuDtTP9nAA39cTdeUJjw9djDdW2sVQxEJj4KkGjmQk8dPXlvGH1du56L+7Xjwcq1iKCLh00+hamLdzsgqhht3H+bui3oz8awuGtorInFBQVINvLfsa+54fSkNExN4ZdIwhmsVQxGJIwqSOJZfUMiDc9cw9ZMNDOrUnKeuGkzbZlrFUETii4IkTmUdOMrNMxcxb8Mexo/ozN0X9dEqhiISlxQkcWjRpr3c+PIi9h3J5eErTuf7g7QAlYjELwVJHHF3Xp6XyX1/WEW7Zg1584Yz6XNK07DLEhEplYIkThzJLeDnby/nzUVbObdXa357xQCaNdICVCIS/xQkcWDT7sNMeTmD1dv386PzenLzud21AJWIVBsKkpB9tHont8xajJkxfcIQvnlq67BLEhE5KQqSkBQWrWL41y/p3TayimGnllrFUESqHwVJCPYdzuVHs5fw0ZosRg3qwP9cplUMRaT6UpBUsZXbIqsYbs/O4b8v7cdVwzppqhMRqdYUJFXojYwt3PXWcpIbJfLqlBEM7JQi7EkGAAAJMUlEQVQcdkkiIhWmIKkCufmF3P+HVcyYl8mIri15/AcDaaVVDEWkhlCQxNjX2Ue48ZVFLN60jylnd+UnF5yqVQxFpEZRkMTQP9bv5uaZiziSW8DvrhrEhf21iqGI1DwKkhhwd579+wYe+OMaUls2YtbkEVrFUERqLAVJJTt4NJ87Xl/K+8u3853+bXnw8tNpolUMRaQG00+4SrRu50GmzEjnq12HuOs7vbjuG101tFdEajwFSSX5YPnX3P7aUhrUS+DlScM4o1ursEsSEakSCpIKyi8o5KG5a3jmkw0M7NScp64aRLtmDcMuS0SkyihIKmDXwaPc/PvF/GPDbq1iKCK1loKknBZv2suNryxiz6Fc/u8/T2fUYK1iKCK1k4LkJLk7r8zfxL3vroysYnjjGfQ9pVnYZYmIhEZBchJy8gr4+VsreGPRFr55agqPXDlQqxiKSK2nIInS5j2HmTIjgy+27+fW83rww3N7aBVDEREUJFH5aM1Obp21BHdn+tVD+GYvrWIoIlJEQVKKwkLn8b+u45EP12oVQxGREsR0rKqZjTSzNWa2zszuLKXdKDNzM0sLtlua2UdmdtDMnjiubaKZTTWztWa22sxGxaL2vIJCJr2Uzm//spbLBrbnjRvOUIiIiJxAzM5IzCwBeBI4H9gCLDSzOe6+6rh2ScAtwPxiu3OAe4B+waO4nwM73b2nmdUBWsSi/noJdejSqjH3X9qPsVrFUESkRLG8tDUUWOfuGwDMbBZwCbDquHb3Aw8APyna4e6HgE/NrPsJ3vdaoFfQrhDYVfmlR9zz3T6xemsRkRojlpe22gObi21vCfYdY2aDgI7u/l40b2hmzYOn95vZIjN7zczaVEq1IiJSLqHN5xFclnoYuO0kDqsLdAA+d/dBwD+A/y3h/SebWbqZpWdlZVW4XhERObFYBslWoGOx7Q7BviJJRO5/fGxmG4HhwJyiG+4l2A0cBt4Mtl8DBp2oobtPdfc0d09LSUkpXw9ERKRMsQyShUAPM+tiZonAaGBO0Yvunu3urdw91d1TgXnAxe6eXtIbursD7wLnBLu+xb/fcxERkSoUs5vt7p5vZjcBc4EEYLq7rzSz+4B0d59T2vHBWUpTINHMLgUuCEZ8/RSYYWaPAFnANbHqg4iIlM0iv+TXbGlpaZ6eXuKJjoiInICZZbh7abcbgBBvtouISM2gIBERkQqpFZe2zCwLyCzn4a2I4Zceq1hN6UtN6QeoL/GqpvSlov3o7O5lDnutFUFSEWaWHs01wuqgpvSlpvQD1Jd4VVP6UlX90KUtERGpEAWJiIhUiIKkbFPDLqAS1ZS+1JR+gPoSr2pKX6qkH7pHIiIiFaIzEhERqRAFSaCs1RzNrL6ZzQ5en29mqVVfZdmi6McEM8sysyXBY1IYdUbDzKab2U4zW1HC62ZmjwV9XRYsSxB3oujHOWaWXewz+UVV1xgtM+sYrF66ysxWmtktJ2gT959LlP2oFp+LmTUwswVmtjToy70naBPbn1/uXusfROYCWw90BRKBpUCf49rcCDwdPB8NzA677nL2YwLwRNi1Rtmf/yAyu/OKEl7/DvABYERmj54fds3l7Mc5wB/CrjPKvrQDBgXPk4C1J/g7FvefS5T9qBafS/D/uUnwvB6R1WaHH9cmpj+/dEYScWw1R3fPBYpWcyzuEuDF4PnrwLcs/tbfjaYf1Ya7fwLsKaXJJcBLHjEPaG5m7aqmuuhF0Y9qw92/dvdFwfMDwBcct2Ad1eBzibIf1ULw//lgsFkveBx/8zumP78UJBFlruZYvI275wPZQMsqqS560fQDYFRwyeF1M+t4gteri2j7Wx2MCC5NfGBmfcMuJhrB5ZGBRH4DLq5afS6l9AOqyediZglmtgTYCfzZ3Uv8TGLx80tBUvu8C6S6+2nAn/nnbykSnkVEpqI4HXgceDvkespkZk2AN4Bb3X1/2PWUVxn9qDafi7sXuPsAIgsIDjWzflX55ytIIspazfFf2phZXaAZkRUb40mZ/XD33e5+NNh8DhhcRbXFQjSfW9xz9/1Flybc/X2gnpm1CrmsEplZPSI/fF9x9zdP0KRafC5l9aO6fS4A7r4P+AgYedxLMf35pSCJKHU1x8Ac4Org+eXAXz24cxVHyuzHcdeqLyZybbi6mgOMD0YJDQey3f3rsIs6WWbWtuh6tZkNJfLvMt5+SQEiI7KAacAX7v5wCc3i/nOJph/V5XMxsxQzax48bwicD6w+rllMf37FbIXE6sSjW81xGpGVGdcRuXE6OryKTyzKfvzQzC4G8on0Y0JoBZfBzGYSGTnTysy2AP9F5EYi7v408D6REULrgMPE6WqZUfTjcuAGM8sHjgCj4/CXlCJnAuOA5cE1eYC7gE5QrT6XaPpRXT6XdsCLZpZAJOxedfc/VOXPL32zXUREKkSXtkREpEIUJCIiUiEKEhERqRAFiYiIVIiCREREKkRBIlIJzOxg2a1KPf51M+saPG9iZs+Y2XozyzCzj81smJklmtknwRfKROKGgkQkZMEcTgnuviHY9RyRsf493H0wke9htAom4vwQuDKcSkVOTEEiUomCb3M/ZGYrzGy5mV0Z7K9jZk+Z2Woz+7OZvW9mlweHXQW8E7TrBgwD7nb3QgB3/8rd3wvavh20F4kbOkUWqVzfBwYApwOtgIVm9gmRb1KnAn2A1kSmppkeHHMmMDN43hdY4u4FJbz/CmBITCoXKSedkYhUrrOAmcFsrDuAvxH5wX8W8Jq7F7r7diIT6xVpB2RF8+ZBwOSaWVIl1y1SbgoSkfAdARoEz1cCpwfzJpWkPpAT86pEoqQgEalcfweuDBYaSiGyzO4C4DMiC4rVMbM2RCZxLPIF0B3A3dcD6cC9xWaeTTWzi4LnLYFd7p5XVR0SKYuCRKRyvQUsA5YCfwXuCC5lvUFkpcBVwMtEFk3KDo55j38NlklAG2Cdma0AXiCy8h3AN4P2InFDs/+KVBEza+LuB4OzigXAme6+PVhD4qNgu6Sb7EXv8SZwp7uvrYKSRaKiUVsiVecPwQJEicD9wZkK7n7EzP6LyLram0o6OFis7G2FiMQbnZGIiEiF6B6JiIhUiIJEREQqREEiIiIVoiAREZEKUZCIiEiFKEhERKRC/j+zACvBzi7s1QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x12041ba90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#visualizaiton\n",
    "n_Cs = len(Cs)\n",
    "n_interest = 3\n",
    "scores = np.zeros((n_interest, n_Cs))\n",
    "\n",
    "for i in range(n_interest):\n",
    "    scores[i][:] = np.mean(lrcv_L1.scores_[i], axis = 0)\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "plt.plot(np.log10(Cs), mse_mean.reshape(n_Cs, 1))\n",
    "plt.xlabel('log(C)')\n",
    "plt.ylabel('neg-logloss')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#L2 with Liblinear\n",
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "Cs = [1, 10, 100, 1000]\n",
    "lrcv_L2 = LogisticRegressionCV(Cs=Cs, cv=5, scoring='neg_log_loss', penalty='l2', solver='liblinear', multi_class='ovr')\n",
    "lrcv_L2.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lrcv_L2.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#visualizaiton\n",
    "n_Cs = len(Cs)\n",
    "n_interest = 3\n",
    "scores = np.zeros((n_interest, n_Cs))\n",
    "\n",
    "for i in range(n_interest):\n",
    "    scores[i][:] = np.mean(lrcv_L2.scores_[i], axis = 0)\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "plt.plot(np.log10(Cs), mse_mean.reshape(n_Cs, 1))\n",
    "plt.xlabel('log(C)')\n",
    "plt.ylabel('neg-logloss')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#L2 with default lbfgs\n",
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "Cs = [1, 10, 100, 1000]\n",
    "lrcv_L2_lbfgs = LogisticRegressionCV(Cs=Cs, cv=5, scoring='neg_log_loss', penalty='l2', multi_class='ovr')\n",
    "lrcv_L2_lbfgs.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lrcv_L2_lbfgs.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_Cs = len(Cs)\n",
    "n_interest = 3\n",
    "scores = np.zeros((n_interest, n_Cs))\n",
    "\n",
    "for i in range(n_interest):\n",
    "    scores[i][:] = np.mean(lrcv_L2_lbfgs.scores_[i], axis = 0)\n",
    "    \n",
    "mse_mean = -np.mean(scores, axis = 0)\n",
    "plt.plot(np.log10(Cs), mse_mean.reshape(n_Cs, 1))\n",
    "plt.xlabel('log(C)')\n",
    "plt.ylabel('neg-logloss')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best model: Logistic Regression with Normalization using GridSearchCV\n",
      "best score:  0.673442223536\n",
      "best params:  {'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "# conclusion\n",
    "print('best model: Logistic Regression with Normalization using GridSearchCV')\n",
    "print('best score: ', -grid.best_score_)\n",
    "print('best params: ', grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_predict = grid.predict(test)\n",
    "predict_df = pd.DataFrame({'listing_id': predict_id, 'interest_level': grid_predict})\n",
    "predict_df.to_csv('./predict/GridsearchCV_predict.csv', columns=['listing_id', 'interest_level'], index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_predict2 = lrcv_L1.predict(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2 2 2 ..., 2 1 2]\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_df = pd.DataFrame({'listing_id': predict_id, 'interest_level': grid_predict2})\n",
    "predict_df.to_csv('./predict/LinearSVC_predict.csv', columns=['listing_id', 'interest_level'], index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
